From a4f9debda8066219dee991b9f4465a99d4cfdfcc Mon Sep 17 00:00:00 2001 From: Jamie Atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 21 Oct 2025 13:38:59 +0200 Subject: [PATCH 01/97] Unify config files to expedition.yaml (#217) Consolidates/unifies the old dual ship_config.yaml and schedule.yaml config files into one expedition.yaml file, in line with v1 dev objectives. --- .github/ISSUE_TEMPLATE/bug_report.md | 2 +- .github/ISSUE_TEMPLATE/feature_request.md | 2 +- docs/user-guide/quickstart.md | 8 +- .../tutorials/Argo_data_tutorial.ipynb | 47 +- src/virtualship/cli/_fetch.py | 28 +- src/virtualship/cli/_plan.py | 1245 ++++++++--------- src/virtualship/cli/commands.py | 35 +- src/virtualship/expedition/do_expedition.py | 64 +- .../expedition/simulate_measurements.py | 23 +- .../expedition/simulate_schedule.py | 90 +- src/virtualship/models/__init__.py | 13 +- src/virtualship/models/expedition.py | 456 ++++++ src/virtualship/models/schedule.py | 236 ---- src/virtualship/models/ship_config.py | 320 ----- src/virtualship/static/expedition.yaml | 75 + src/virtualship/static/schedule.yaml | 42 - src/virtualship/static/ship_config.yaml | 30 - src/virtualship/utils.py | 63 +- tests/cli/test_cli.py | 25 +- tests/cli/test_fetch.py | 28 +- tests/cli/test_plan.py | 30 +- .../expedition/expedition_dir/expedition.yaml | 46 + tests/expedition/expedition_dir/schedule.yaml | 18 - .../expedition_dir/ship_config.yaml | 25 - tests/expedition/test_expedition.py | 277 ++++ tests/expedition/test_schedule.py | 160 --- tests/expedition/test_ship_config.py | 126 -- tests/expedition/test_simulate_schedule.py | 31 +- tests/test_mfp_to_yaml.py | 12 +- tests/test_utils.py | 26 +- 30 files changed, 1700 insertions(+), 1883 deletions(-) create mode 100644 src/virtualship/models/expedition.py delete mode 100644 src/virtualship/models/schedule.py delete mode 100644 src/virtualship/models/ship_config.py create mode 100644 src/virtualship/static/expedition.yaml delete mode 100644 src/virtualship/static/schedule.yaml delete mode 100644 src/virtualship/static/ship_config.yaml create mode 100644 tests/expedition/expedition_dir/expedition.yaml delete mode 100644 tests/expedition/expedition_dir/schedule.yaml delete mode 100644 tests/expedition/expedition_dir/ship_config.yaml create mode 100644 tests/expedition/test_expedition.py delete mode 100644 tests/expedition/test_schedule.py delete mode 100644 tests/expedition/test_ship_config.py diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 138cbb37e..b53734705 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,7 +1,7 @@ --- name: Bug report about: Create a report to help us improve -title: "" +title: ["needs-triage"] labels: bug assignees: "" --- diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 8e47557f9..1adc441f8 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -1,7 +1,7 @@ --- name: Feature request about: Suggest an idea for this project -title: "" +title: ["needs-triage"] labels: enhancement assignees: "" --- diff --git a/docs/user-guide/quickstart.md b/docs/user-guide/quickstart.md index 59a514c75..45d4050f3 100644 --- a/docs/user-guide/quickstart.md +++ b/docs/user-guide/quickstart.md @@ -46,10 +46,10 @@ virtualship init EXPEDITION_NAME --from-mfp CoordinatesExport.xlsx The `CoordinatesExport.xlsx` in the `virtualship init` command refers to the .xlsx file exported from MFP. Replace the filename with the name of your exported .xlsx file (and make sure to move it from the Downloads to the folder/directory in which you are running the expedition). ``` -This will create a folder/directory called `EXPEDITION_NAME` with two files: `schedule.yaml` and `ship_config.yaml` based on the sampling site coordinates that you specified in your MFP export. The `--from-mfp` flag indictates that the exported coordinates will be used. +This will create a folder/directory called `EXPEDITION_NAME` with a single file: `expedition.yaml` containing details on the ship and instrument configurations, as well as the expedition schedule based on the sampling site coordinates that you specified in your MFP export. The `--from-mfp` flag indicates that the exported coordinates will be used. ```{note} -For advanced users: it is also possible to run the expedition initialisation step without an MFP .xlsx export file. In this case you should simply run `virtualship init EXPEDITION_NAME` in the CLI. This will write example `schedule.yaml` and `ship_config.yaml` files in the `EXPEDITION_NAME` folder/directory. These files contain example waypoints, timings and instrument selections, but can be edited or propagated through the rest of the workflow unedited to run a sample expedition. +For advanced users: it is also possible to run the expedition initialisation step without an MFP .xlsx export file. In this case you should simply run `virtualship init EXPEDITION_NAME` in the CLI. This will write an example `expedition.yaml` file in the `EXPEDITION_NAME` folder/directory. This file contains example waypoints, timings, instrument selections, and ship configuration, but can be edited or propagated through the rest of the workflow unedited to run a sample expedition. ``` ## Expedition scheduling & ship configuration @@ -61,7 +61,7 @@ virtualship plan EXPEDITION_NAME ``` ```{tip} -Using the `virtualship plan` tool is optional. Advanced users can also edit the `schedule.yaml` and `ship_config.yaml` files directly if preferred. +Using the `virtualship plan` tool is optional. Advanced users can also edit the `expedition.yaml` file directly if preferred. ``` The planning tool should look something like this and offers an intuitive way to make your selections: @@ -111,7 +111,7 @@ For advanced users: you can also make further customisations to behaviours of al When you are happy with your ship configuration and schedule plan, press _Save Changes_. ```{note} -On pressing _Save Changes_ the tool will check the selections are valid (for example that the ship will be able to reach each waypoint in time). If they are, the changes will be saved to the `ship_config.yaml` and `schedule.yaml` files, ready for the next steps. If your selections are invalid you should be provided with information on how to fix them. +On pressing _Save Changes_ the tool will check the selections are valid (for example that the ship will be able to reach each waypoint in time). If they are, the changes will be saved to the `expedition.yaml` file, ready for the next steps. If your selections are invalid you should be provided with information on how to fix them. ``` ## Fetch the data diff --git a/docs/user-guide/tutorials/Argo_data_tutorial.ipynb b/docs/user-guide/tutorials/Argo_data_tutorial.ipynb index 30cee4609..e82353151 100644 --- a/docs/user-guide/tutorials/Argo_data_tutorial.ipynb +++ b/docs/user-guide/tutorials/Argo_data_tutorial.ipynb @@ -14,7 +14,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -28,25 +28,26 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We have downloaded the data from Copernicus Marine Service, using `virtualship fetch` and the information in following `schedule.yaml` file:\n", + "We have downloaded the data from Copernicus Marine Service, using `virtualship fetch` and the information in following `schedule` section of the `expedition.yaml` file:\n", "```yaml\n", - "space_time_region:\n", - " spatial_range:\n", - " minimum_longitude: -5\n", - " maximum_longitude: 5\n", - " minimum_latitude: -5\n", - " maximum_latitude: 5\n", - " minimum_depth: 0\n", - " maximum_depth: 2000\n", - " time_range:\n", - " start_time: 2023-01-01 00:00:00\n", - " end_time: 2023-02-01 00:00:00\n", - "waypoints:\n", - " - instrument: ARGO_FLOAT\n", - " location:\n", - " latitude: 0.02\n", - " longitude: 0.02\n", - " time: 2023-01-01 02:00:00\n", + "schedule:\n", + " space_time_region:\n", + " spatial_range:\n", + " minimum_longitude: -5\n", + " maximum_longitude: 5\n", + " minimum_latitude: -5\n", + " maximum_latitude: 5\n", + " minimum_depth: 0\n", + " maximum_depth: 2000\n", + " time_range:\n", + " start_time: 2023-01-01 00:00:00\n", + " end_time: 2023-02-01 00:00:00\n", + " waypoints:\n", + " - instrument: ARGO_FLOAT\n", + " location:\n", + " latitude: 0.02\n", + " longitude: 0.02\n", + " time: 2023-01-01 02:00:00\n", "```\n", "\n", "After running `virtualship run`, we have a `results/argo_floats.zarr` file with the data from the float." @@ -54,7 +55,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -79,7 +80,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -111,7 +112,7 @@ ], "metadata": { "kernelspec": { - "display_name": "parcels", + "display_name": "ship", "language": "python", "name": "python3" }, @@ -125,7 +126,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.3" + "version": "3.12.9" } }, "nbformat": 4, diff --git a/src/virtualship/cli/_fetch.py b/src/virtualship/cli/_fetch.py index ac039d761..600083040 100644 --- a/src/virtualship/cli/_fetch.py +++ b/src/virtualship/cli/_fetch.py @@ -12,8 +12,7 @@ from virtualship.utils import ( _dump_yaml, _generic_load_yaml, - _get_schedule, - _get_ship_config, + _get_expedition, ) if TYPE_CHECKING: @@ -24,7 +23,7 @@ from copernicusmarine.core_functions.credentials_utils import InvalidUsernameOrPassword import virtualship.cli._creds as creds -from virtualship.utils import SCHEDULE +from virtualship.utils import EXPEDITION DOWNLOAD_METADATA = "download_metadata.yaml" @@ -49,17 +48,18 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None data_folder = path / "data" data_folder.mkdir(exist_ok=True) - schedule = _get_schedule(path) - ship_config = _get_ship_config(path) + expedition = _get_expedition(path) - schedule.verify( - ship_config.ship_speed_knots, + expedition.schedule.verify( + expedition.ship_config.ship_speed_knots, input_data=None, check_space_time_region=True, ignore_missing_fieldsets=True, ) - space_time_region_hash = get_space_time_region_hash(schedule.space_time_region) + space_time_region_hash = get_space_time_region_hash( + expedition.schedule.space_time_region + ) existing_download = get_existing_download(data_folder, space_time_region_hash) if existing_download is not None: @@ -72,11 +72,11 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None username, password = creds.get_credentials_flow(username, password, creds_path) # Extract space_time_region details from the schedule - spatial_range = schedule.space_time_region.spatial_range - time_range = schedule.space_time_region.time_range + spatial_range = expedition.schedule.space_time_region.spatial_range + time_range = expedition.schedule.space_time_region.time_range start_datetime = time_range.start_time end_datetime = time_range.end_time - instruments_in_schedule = schedule.get_instruments() + instruments_in_schedule = expedition.schedule.get_instruments() # Create download folder and set download metadata download_folder = data_folder / hash_to_filename(space_time_region_hash) @@ -84,15 +84,15 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None DownloadMetadata(download_complete=False).to_yaml( download_folder / DOWNLOAD_METADATA ) - shutil.copyfile(path / SCHEDULE, download_folder / SCHEDULE) + shutil.copyfile(path / EXPEDITION, download_folder / EXPEDITION) if ( ( {"XBT", "CTD", "CDT_BGC", "SHIP_UNDERWATER_ST"} & set(instrument.name for instrument in instruments_in_schedule) ) - or ship_config.ship_underwater_st_config is not None - or ship_config.adcp_config is not None + or expedition.instruments_config.ship_underwater_st_config is not None + or expedition.instruments_config.adcp_config is not None ): print("Ship data will be downloaded. Please wait...") diff --git a/src/virtualship/cli/_plan.py b/src/virtualship/cli/_plan.py index 85539e3fc..87bfe3363 100644 --- a/src/virtualship/cli/_plan.py +++ b/src/virtualship/cli/_plan.py @@ -1,7 +1,6 @@ import datetime import os import traceback -from typing import ClassVar from textual import on from textual.app import App, ComposeResult @@ -30,23 +29,23 @@ type_to_textual, ) from virtualship.errors import UnexpectedError, UserError -from virtualship.models.location import Location -from virtualship.models.schedule import Schedule, Waypoint -from virtualship.models.ship_config import ( +from virtualship.models import ( ADCPConfig, ArgoFloatConfig, CTD_BGCConfig, CTDConfig, DrifterConfig, + Expedition, InstrumentType, + Location, ShipConfig, ShipUnderwaterSTConfig, - XBTConfig, -) -from virtualship.models.space_time_region import ( SpatialRange, TimeRange, + Waypoint, + XBTConfig, ) +from virtualship.utils import EXPEDITION UNEXPECTED_MSG_ONSAVE = ( "Please ensure that:\n" @@ -81,227 +80,236 @@ def log_exception_to_file( f.write("\n") -class WaypointWidget(Static): - def __init__(self, waypoint: Waypoint, index: int): +DEFAULT_TS_CONFIG = {"period_minutes": 5.0} + +DEFAULT_ADCP_CONFIG = { + "num_bins": 40, + "period_minutes": 5.0, +} + +INSTRUMENT_FIELDS = { + "adcp_config": { + "class": ADCPConfig, + "title": "Onboard ADCP", + "attributes": [ + {"name": "num_bins"}, + {"name": "period", "minutes": True}, + ], + }, + "ship_underwater_st_config": { + "class": ShipUnderwaterSTConfig, + "title": "Onboard Temperature/Salinity", + "attributes": [ + {"name": "period", "minutes": True}, + ], + }, + "ctd_config": { + "class": CTDConfig, + "title": "CTD", + "attributes": [ + {"name": "max_depth_meter"}, + {"name": "min_depth_meter"}, + {"name": "stationkeeping_time", "minutes": True}, + ], + }, + "ctd_bgc_config": { + "class": CTD_BGCConfig, + "title": "CTD-BGC", + "attributes": [ + {"name": "max_depth_meter"}, + {"name": "min_depth_meter"}, + {"name": "stationkeeping_time", "minutes": True}, + ], + }, + "xbt_config": { + "class": XBTConfig, + "title": "XBT", + "attributes": [ + {"name": "min_depth_meter"}, + {"name": "max_depth_meter"}, + {"name": "fall_speed_meter_per_second"}, + {"name": "deceleration_coefficient"}, + ], + }, + "argo_float_config": { + "class": ArgoFloatConfig, + "title": "Argo Float", + "attributes": [ + {"name": "min_depth_meter"}, + {"name": "max_depth_meter"}, + {"name": "drift_depth_meter"}, + {"name": "vertical_speed_meter_per_second"}, + {"name": "cycle_days"}, + {"name": "drift_days"}, + ], + }, + "drifter_config": { + "class": DrifterConfig, + "title": "Drifter", + "attributes": [ + {"name": "depth_meter"}, + {"name": "lifetime", "minutes": True}, + ], + }, +} + + +class ExpeditionEditor(Static): + def __init__(self, path: str): super().__init__() - self.waypoint = waypoint - self.index = index + self.path = path + self.expedition = None def compose(self) -> ComposeResult: try: - with Collapsible( - title=f"[b]Waypoint {self.index + 1}[/b]", - collapsed=True, - id=f"wp{self.index + 1}", - ): - if self.index > 0: - yield Button( - "Copy Time & Instruments from Previous", - id=f"wp{self.index}_copy", - variant="warning", - ) - yield Label("Location:") - yield Label(" Latitude:") - yield Input( - id=f"wp{self.index}_lat", - value=str(self.waypoint.location.lat) - if self.waypoint.location.lat - is not None # is not None to handle if lat is 0.0 - else "", - validators=[ - Function( - is_valid_lat, - f"INVALID: value must be {is_valid_lat.__doc__.lower()}", - ) - ], - type="number", - placeholder="°N", - classes="latitude-input", - ) - yield Label( - "", - id=f"validation-failure-label-wp{self.index}_lat", - classes="-hidden validation-failure", - ) + self.expedition = Expedition.from_yaml(self.path.joinpath(EXPEDITION)) + except Exception as e: + raise UserError( + f"There is an issue in {self.path.joinpath(EXPEDITION)}:\n\n{e}" + ) from None - yield Label(" Longitude:") - yield Input( - id=f"wp{self.index}_lon", - value=str(self.waypoint.location.lon) - if self.waypoint.location.lon - is not None # is not None to handle if lon is 0.0 - else "", - validators=[ - Function( - is_valid_lon, - f"INVALID: value must be {is_valid_lon.__doc__.lower()}", - ) - ], - type="number", - placeholder="°E", - classes="longitude-input", - ) - yield Label( - "", - id=f"validation-failure-label-wp{self.index}_lon", - classes="-hidden validation-failure", - ) + try: + ## 1) SHIP SPEED & INSTRUMENTS CONFIG EDITOR - yield Label("Time:") - with Horizontal(): - yield Label("Year:") - yield Select( - [ - (str(year), year) - # TODO: change from hard coding? ...flexibility for different datasets... - for year in range( - 2022, - datetime.datetime.now().year + 1, + yield Label( + "[b]Ship & Instruments Config Editor[/b]", + id="title_ship_instruments_config", + markup=True, + ) + yield Rule(line_style="heavy") + + # SECTION: "Ship Speed & Onboard Measurements" + + with Collapsible( + title="[b]Ship Speed & Onboard Measurements[/b]", + id="speed_collapsible", + collapsed=False, + ): + attr = "ship_speed_knots" + validators = group_validators(ShipConfig, attr) + with Horizontal(classes="ship_speed"): + yield Label("[b]Ship Speed (knots):[/b]") + yield Input( + id="speed", + type=type_to_textual(get_field_type(ShipConfig, attr)), + validators=[ + Function( + validator, + f"INVALID: value must be {validator.__doc__.lower()}", ) + for validator in validators ], - id=f"wp{self.index}_year", - value=int(self.waypoint.time.year) - if self.waypoint.time - else Select.BLANK, - prompt="YYYY", - classes="year-select", - ) - yield Label("Month:") - yield Select( - [(f"{m:02d}", m) for m in range(1, 13)], - id=f"wp{self.index}_month", - value=int(self.waypoint.time.month) - if self.waypoint.time - else Select.BLANK, - prompt="MM", - classes="month-select", + classes="ship_speed_input", + placeholder="knots", + value=str( + self.expedition.ship_config.ship_speed_knots + if self.expedition.ship_config.ship_speed_knots + else "" + ), ) - yield Label("Day:") - yield Select( - [(f"{d:02d}", d) for d in range(1, 32)], - id=f"wp{self.index}_day", - value=int(self.waypoint.time.day) - if self.waypoint.time - else Select.BLANK, - prompt="DD", - classes="day-select", + yield Label("", id="validation-failure-label-speed", classes="-hidden") + + with Horizontal(classes="ts-section"): + yield Label("[b]Onboard Temperature/Salinity:[/b]") + yield Switch( + value=bool( + self.expedition.instruments_config.ship_underwater_st_config + ), + id="has_onboard_ts", ) - yield Label("Hour:") - yield Select( - [(f"{h:02d}", h) for h in range(24)], - id=f"wp{self.index}_hour", - value=int(self.waypoint.time.hour) - if self.waypoint.time - else Select.BLANK, - prompt="hh", - classes="hour-select", + + with Horizontal(classes="adcp-section"): + yield Label("[b]Onboard ADCP:[/b]") + yield Switch( + value=bool(self.expedition.instruments_config.adcp_config), + id="has_adcp", ) - yield Label("Min:") - yield Select( - [(f"{m:02d}", m) for m in range(0, 60, 5)], - id=f"wp{self.index}_minute", - value=int(self.waypoint.time.minute) - if self.waypoint.time - else Select.BLANK, - prompt="mm", - classes="minute-select", + + # adcp type selection + with Horizontal(id="adcp_type_container", classes="-hidden"): + is_deep = ( + self.expedition.instruments_config.adcp_config + and self.expedition.instruments_config.adcp_config.max_depth_meter + == -1000.0 ) + yield Label(" OceanObserver:") + yield Switch(value=is_deep, id="adcp_deep") + yield Label(" SeaSeven:") + yield Switch(value=not is_deep, id="adcp_shallow") + yield Button("?", id="info_button", variant="warning") - yield Label("Instruments:") - for instrument in InstrumentType: - is_selected = instrument in (self.waypoint.instrument or []) - with Horizontal(): - yield Label(instrument.value) - yield Switch( - value=is_selected, id=f"wp{self.index}_{instrument.value}" - ) + ## SECTION: "Instrument Configurations"" - if instrument.value == "DRIFTER": - yield Label("Count") - yield Input( - id=f"wp{self.index}_drifter_count", - value=str( - self.get_drifter_count() if is_selected else "" - ), - type="integer", - placeholder="# of drifters", - validators=Integer( - minimum=1, - failure_description="INVALID: value must be > 0", - ), - classes="drifter-count-input", - ) + with Collapsible( + title="[b]Instrument Configurations[/b] (advanced users only)", + collapsed=True, + ): + for instrument_name, info in INSTRUMENT_FIELDS.items(): + config_class = info["class"] + attributes = info["attributes"] + # instrument-specific configs now live under instruments_config + config_instance = getattr( + self.expedition.instruments_config, instrument_name, None + ) + title = info.get("title", instrument_name.replace("_", " ").title()) + with Collapsible( + title=f"[b]{title}[/b]", + collapsed=True, + ): + if instrument_name in ( + "adcp_config", + "ship_underwater_st_config", + ): yield Label( - "", - id=f"validation-failure-label-wp{self.index}_drifter_count", - classes="-hidden validation-failure", + f"NOTE: entries will be ignored here if {info['title']} is OFF in Ship Speed & Onboard Measurements." ) + with Container(classes="instrument-config"): + for attr_meta in attributes: + attr = attr_meta["name"] + is_minutes = attr_meta.get("minutes", False) + validators = group_validators(config_class, attr) + if config_instance: + raw_value = getattr(config_instance, attr, "") + if is_minutes and raw_value != "": + try: + value = str( + raw_value.total_seconds() / 60.0 + ) + except AttributeError: + value = str(raw_value) + else: + value = str(raw_value) + else: + value = "" + label = f"{attr.replace('_', ' ').title()}:" + yield Label( + label + if not is_minutes + else label.replace(":", " Minutes:") + ) + yield Input( + id=f"{instrument_name}_{attr}", + type=type_to_textual( + get_field_type(config_class, attr) + ), + validators=[ + Function( + validator, + f"INVALID: value must be {validator.__doc__.lower()}", + ) + for validator in validators + ], + value=value, + ) + yield Label( + "", + id=f"validation-failure-label-{instrument_name}_{attr}", + classes="-hidden validation-failure", + ) - except Exception as e: - raise UnexpectedError(unexpected_msg_compose(e)) from None - - def get_drifter_count(self) -> int: - return sum( - 1 for inst in self.waypoint.instrument if inst == InstrumentType.DRIFTER - ) - - def copy_from_previous(self) -> None: - """Copy inputs from previous waypoint widget (time and instruments only, not lat/lon).""" - try: - if self.index > 0: - schedule_editor = self.parent - if schedule_editor: - time_components = ["year", "month", "day", "hour", "minute"] - for comp in time_components: - prev = schedule_editor.query_one(f"#wp{self.index - 1}_{comp}") - curr = self.query_one(f"#wp{self.index}_{comp}") - if prev and curr: - curr.value = prev.value - - for instrument in InstrumentType: - prev_switch = schedule_editor.query_one( - f"#wp{self.index - 1}_{instrument.value}" - ) - curr_switch = self.query_one( - f"#wp{self.index}_{instrument.value}" - ) - if prev_switch and curr_switch: - curr_switch.value = prev_switch.value - except Exception as e: - raise UnexpectedError(unexpected_msg_compose(e)) from None - - @on(Button.Pressed, "Button") - def button_pressed(self, event: Button.Pressed) -> None: - if event.button.id == f"wp{self.index}_copy": - self.copy_from_previous() - - @on(Switch.Changed) - def on_switch_changed(self, event: Switch.Changed) -> None: - if event.switch.id == f"wp{self.index}_DRIFTER": - drifter_count_input = self.query_one( - f"#wp{self.index}_drifter_count", Input - ) - if not event.value: - drifter_count_input.value = "" - else: - if not drifter_count_input.value: - drifter_count_input.value = "1" - - -class ScheduleEditor(Static): - def __init__(self, path: str): - super().__init__() - self.path = path - self.schedule = None - - def compose(self) -> ComposeResult: - try: - self.schedule = Schedule.from_yaml(f"{self.path}/schedule.yaml") - except Exception as e: - raise UserError(f"There is an issue in schedule.yaml:\n\n{e}") from None + ## 2) SCHEDULE EDITOR - try: - yield Label("[b]Schedule Editor[/b]", id="title", markup=True) + yield Label("[b]Schedule Editor[/b]", id="title_schedule", markup=True) yield Rule(line_style="heavy") # SECTION: "Waypoints & Instrument Selection" @@ -327,8 +335,8 @@ def compose(self) -> ComposeResult: title="[b]Space-Time Region[/b] (advanced users only)", collapsed=True, ): - if self.schedule.space_time_region: - str_data = self.schedule.space_time_region + if self.expedition.schedule.space_time_region: + str_data = self.expedition.schedule.space_time_region yield Label("Minimum Latitude:") yield Input( @@ -501,13 +509,137 @@ def compose(self) -> ComposeResult: def on_mount(self) -> None: self.refresh_waypoint_widgets() + adcp_present = ( + getattr(self.expedition.instruments_config, "adcp_config", None) + if self.expedition.instruments_config + else False + ) + self.show_hide_adcp_type(bool(adcp_present)) def refresh_waypoint_widgets(self): waypoint_list = self.query_one("#waypoint_list", VerticalScroll) waypoint_list.remove_children() - for i, waypoint in enumerate(self.schedule.waypoints): + for i, waypoint in enumerate(self.expedition.schedule.waypoints): waypoint_list.mount(WaypointWidget(waypoint, i)) + def save_changes(self) -> bool: + """Save changes to expedition.yaml.""" + try: + self._update_ship_speed() + self._update_instrument_configs() + self._update_schedule() + self.expedition.to_yaml(self.path.joinpath(EXPEDITION)) + return True + except Exception as e: + log_exception_to_file( + e, + self.path, + context_message=f"Error saving {self.path.joinpath(EXPEDITION)}:", + ) + raise UnexpectedError( + UNEXPECTED_MSG_ONSAVE + + f"\n\nTraceback will be logged in {self.path}/virtualship_error.txt. Please attach this/copy the contents to any issue submitted." + ) from None + + def _update_ship_speed(self): + attr = "ship_speed_knots" + field_type = get_field_type(type(self.expedition.ship_config), attr) + value = field_type(self.query_one("#speed").value) + ShipConfig.model_validate( + {**self.expedition.ship_config.model_dump(), attr: value} + ) + self.expedition.ship_config.ship_speed_knots = value + + def _update_instrument_configs(self): + for instrument_name, info in INSTRUMENT_FIELDS.items(): + config_class = info["class"] + attributes = info["attributes"] + kwargs = {} + # special handling for onboard ADCP and T/S + if instrument_name == "adcp_config": + has_adcp = self.query_one("#has_adcp", Switch).value + if not has_adcp: + setattr(self.expedition.instruments_config, instrument_name, None) + continue + if instrument_name == "ship_underwater_st_config": + has_ts = self.query_one("#has_onboard_ts", Switch).value + if not has_ts: + setattr(self.expedition.instruments_config, instrument_name, None) + continue + for attr_meta in attributes: + attr = attr_meta["name"] + is_minutes = attr_meta.get("minutes", False) + input_id = f"{instrument_name}_{attr}" + value = self.query_one(f"#{input_id}").value + field_type = get_field_type(config_class, attr) + if is_minutes and field_type is datetime.timedelta: + value = datetime.timedelta(minutes=float(value)) + else: + value = field_type(value) + kwargs[attr] = value + # ADCP max_depth_meter based on deep/shallow switch + if instrument_name == "adcp_config": + if self.query_one("#adcp_deep", Switch).value: + kwargs["max_depth_meter"] = -1000.0 + else: + kwargs["max_depth_meter"] = -150.0 + setattr( + self.expedition.instruments_config, + instrument_name, + config_class(**kwargs), + ) + + def _update_schedule(self): + spatial_range = SpatialRange( + minimum_longitude=self.query_one("#min_lon").value, + maximum_longitude=self.query_one("#max_lon").value, + minimum_latitude=self.query_one("#min_lat").value, + maximum_latitude=self.query_one("#max_lat").value, + minimum_depth=self.query_one("#min_depth").value, + maximum_depth=self.query_one("#max_depth").value, + ) + start_time_input = self.query_one("#start_time").value + end_time_input = self.query_one("#end_time").value + waypoint_times = [ + wp.time + for wp in self.expedition.schedule.waypoints + if hasattr(wp, "time") and wp.time + ] + if not start_time_input and waypoint_times: + start_time = min(waypoint_times) + else: + start_time = start_time_input + if not end_time_input and waypoint_times: + end_time = max(waypoint_times) + datetime.timedelta(minutes=60480.0) + else: + end_time = end_time_input + time_range = TimeRange(start_time=start_time, end_time=end_time) + self.expedition.schedule.space_time_region.spatial_range = spatial_range + self.expedition.schedule.space_time_region.time_range = time_range + for i, wp in enumerate(self.expedition.schedule.waypoints): + wp.location = Location( + latitude=float(self.query_one(f"#wp{i}_lat").value), + longitude=float(self.query_one(f"#wp{i}_lon").value), + ) + wp.time = datetime.datetime( + int(self.query_one(f"#wp{i}_year").value), + int(self.query_one(f"#wp{i}_month").value), + int(self.query_one(f"#wp{i}_day").value), + int(self.query_one(f"#wp{i}_hour").value), + int(self.query_one(f"#wp{i}_minute").value), + 0, + ) + wp.instrument = [] + for instrument in InstrumentType: + switch_on = self.query_one(f"#wp{i}_{instrument.value}").value + if instrument.value == "DRIFTER" and switch_on: + count_str = self.query_one(f"#wp{i}_drifter_count").value + count = int(count_str) + assert count > 0 + wp.instrument.extend([InstrumentType.DRIFTER] * count) + elif switch_on: + wp.instrument.append(instrument) + @on(Input.Changed) def show_invalid_reasons(self, event: Input.Changed) -> None: input_id = event.input.id @@ -547,8 +679,8 @@ def show_invalid_reasons(self, event: Input.Changed) -> None: def add_waypoint(self) -> None: """Add a new waypoint to the schedule. Copies time from last waypoint if possible (Lat/lon and instruments blank).""" try: - if self.schedule.waypoints: - last_wp = self.schedule.waypoints[-1] + if self.expedition.schedule.waypoints: + last_wp = self.expedition.schedule.waypoints[-1] new_time = last_wp.time if last_wp.time else None new_wp = Waypoint( location=Location( @@ -558,320 +690,27 @@ def add_waypoint(self) -> None: time=new_time, instrument=[], ) - else: - new_wp = Waypoint( - location=Location(latitude=0.0, longitude=0.0), - time=None, - instrument=[], - ) - self.schedule.waypoints.append(new_wp) - self.refresh_waypoint_widgets() - - except Exception as e: - raise UnexpectedError(unexpected_msg_compose(e)) from None - - @on(Button.Pressed, "#remove_waypoint") - def remove_waypoint(self) -> None: - """Remove the last waypoint from the schedule.""" - try: - if self.schedule.waypoints: - self.schedule.waypoints.pop() - self.refresh_waypoint_widgets() - else: - self.notify("No waypoints to remove.", severity="error", timeout=5) - - except Exception as e: - raise UnexpectedError(unexpected_msg_compose(e)) from None - - def save_changes(self) -> bool: - """Save changes to schedule.yaml.""" - try: - ## spacetime region - spatial_range = SpatialRange( - minimum_longitude=self.query_one("#min_lon").value, - maximum_longitude=self.query_one("#max_lon").value, - minimum_latitude=self.query_one("#min_lat").value, - maximum_latitude=self.query_one("#max_lat").value, - minimum_depth=self.query_one("#min_depth").value, - maximum_depth=self.query_one("#max_depth").value, - ) - - # auto fill start and end times if input is blank - start_time_input = self.query_one("#start_time").value - end_time_input = self.query_one("#end_time").value - waypoint_times = [ - wp.time - for wp in self.schedule.waypoints - if hasattr(wp, "time") and wp.time - ] - - if not start_time_input and waypoint_times: - start_time = min(waypoint_times) - else: - start_time = start_time_input - - if not end_time_input and waypoint_times: - end_time = max(waypoint_times) + datetime.timedelta( - minutes=60480.0 - ) # with buffer (corresponds to default drifter lifetime) - else: - end_time = end_time_input - - time_range = TimeRange( - start_time=start_time, - end_time=end_time, - ) - - self.schedule.space_time_region.spatial_range = spatial_range - self.schedule.space_time_region.time_range = time_range - - ## waypoints - for i, wp in enumerate(self.schedule.waypoints): - wp.location = Location( - latitude=float(self.query_one(f"#wp{i}_lat").value), - longitude=float(self.query_one(f"#wp{i}_lon").value), - ) - wp.time = datetime.datetime( - int(self.query_one(f"#wp{i}_year").value), - int(self.query_one(f"#wp{i}_month").value), - int(self.query_one(f"#wp{i}_day").value), - int(self.query_one(f"#wp{i}_hour").value), - int(self.query_one(f"#wp{i}_minute").value), - 0, - ) - - wp.instrument = [] - for instrument in InstrumentType: - switch_on = self.query_one(f"#wp{i}_{instrument.value}").value - if instrument.value == "DRIFTER" and switch_on: - count_str = self.query_one(f"#wp{i}_drifter_count").value - count = int(count_str) - assert count > 0 - wp.instrument.extend([InstrumentType.DRIFTER] * count) - elif switch_on: - wp.instrument.append(instrument) - - # save - self.schedule.to_yaml(f"{self.path}/schedule.yaml") - return True - - except Exception as e: - log_exception_to_file( - e, self.path, context_message="Error saving schedule:" - ) - - raise UnexpectedError( - UNEXPECTED_MSG_ONSAVE - + f"\n\nTraceback will be logged in {self.path}/virtualship_error.txt. Please attach this/copy the contents to any issue submitted." - ) from None - - -class ConfigEditor(Container): - DEFAULT_ADCP_CONFIG: ClassVar[dict[str, float]] = { - "num_bins": 40, - "period_minutes": 5.0, - } - - DEFAULT_TS_CONFIG: ClassVar[dict[str, float]] = {"period_minutes": 5.0} - - INSTRUMENT_FIELDS: ClassVar[dict[str, dict]] = { - "adcp_config": { - "class": ADCPConfig, - "title": "Onboard ADCP", - "attributes": [ - {"name": "num_bins"}, - {"name": "period", "minutes": True}, - ], - }, - "ship_underwater_st_config": { - "class": ShipUnderwaterSTConfig, - "title": "Onboard Temperature/Salinity", - "attributes": [ - {"name": "period", "minutes": True}, - ], - }, - "ctd_config": { - "class": CTDConfig, - "title": "CTD", - "attributes": [ - {"name": "max_depth_meter"}, - {"name": "min_depth_meter"}, - {"name": "stationkeeping_time", "minutes": True}, - ], - }, - "ctd_bgc_config": { - "class": CTD_BGCConfig, - "title": "CTD-BGC", - "attributes": [ - {"name": "max_depth_meter"}, - {"name": "min_depth_meter"}, - {"name": "stationkeeping_time", "minutes": True}, - ], - }, - "xbt_config": { - "class": XBTConfig, - "title": "XBT", - "attributes": [ - {"name": "min_depth_meter"}, - {"name": "max_depth_meter"}, - {"name": "fall_speed_meter_per_second"}, - {"name": "deceleration_coefficient"}, - ], - }, - "argo_float_config": { - "class": ArgoFloatConfig, - "title": "Argo Float", - "attributes": [ - {"name": "min_depth_meter"}, - {"name": "max_depth_meter"}, - {"name": "drift_depth_meter"}, - {"name": "vertical_speed_meter_per_second"}, - {"name": "cycle_days"}, - {"name": "drift_days"}, - ], - }, - "drifter_config": { - "class": DrifterConfig, - "title": "Drifter", - "attributes": [ - {"name": "depth_meter"}, - {"name": "lifetime", "minutes": True}, - ], - }, - } - - def __init__(self, path: str): - super().__init__() - self.path = path - self.config = None - - def compose(self) -> ComposeResult: - try: - self.config = ShipConfig.from_yaml(f"{self.path}/ship_config.yaml") - except Exception as e: - raise UserError(f"There is an issue in ship_config.yaml:\n\n{e}") from None - - try: - ## SECTION: "Ship Speed & Onboard Measurements" - - yield Label("[b]Ship Config Editor[/b]", id="title", markup=True) - yield Rule(line_style="heavy") - - with Collapsible( - title="[b]Ship Speed & Onboard Measurements[/b]", id="speed_collapsible" - ): - attr = "ship_speed_knots" - validators = group_validators(ShipConfig, attr) - with Horizontal(classes="ship_speed"): - yield Label("[b]Ship Speed (knots):[/b]") - yield Input( - id="speed", - type=type_to_textual(get_field_type(ShipConfig, attr)), - validators=[ - Function( - validator, - f"INVALID: value must be {validator.__doc__.lower()}", - ) - for validator in validators - ], - classes="ship_speed_input", - placeholder="knots", - value=str( - self.config.ship_speed_knots - if self.config.ship_speed_knots - else "" - ), - ) - yield Label("", id="validation-failure-label-speed", classes="-hidden") - - with Horizontal(classes="ts-section"): - yield Label("[b]Onboard Temperature/Salinity:[/b]") - yield Switch( - value=bool(self.config.ship_underwater_st_config), - id="has_onboard_ts", - ) - - with Horizontal(classes="adcp-section"): - yield Label("[b]Onboard ADCP:[/b]") - yield Switch(value=bool(self.config.adcp_config), id="has_adcp") - - # adcp type selection - with Horizontal(id="adcp_type_container", classes="-hidden"): - is_deep = ( - self.config.adcp_config - and self.config.adcp_config.max_depth_meter == -1000.0 - ) - yield Label(" OceanObserver:") - yield Switch(value=is_deep, id="adcp_deep") - yield Label(" SeaSeven:") - yield Switch(value=not is_deep, id="adcp_shallow") - yield Button("?", id="info_button", variant="warning") - - ## SECTION: "Instrument Configurations"" - - with Collapsible( - title="[b]Instrument Configurations[/b] (advanced users only)", - collapsed=True, - ): - for instrument_name, info in self.INSTRUMENT_FIELDS.items(): - config_class = info["class"] - attributes = info["attributes"] - config_instance = getattr(self.config, instrument_name, None) - title = info.get("title", instrument_name.replace("_", " ").title()) - with Collapsible( - title=f"[b]{title}[/b]", - collapsed=True, - ): - if instrument_name in ( - "adcp_config", - "ship_underwater_st_config", - ): - yield Label( - f"NOTE: entries will be ignored here if {info['title']} is OFF in Ship Speed & Onboard Measurements." - ) - with Container(classes="instrument-config"): - for attr_meta in attributes: - attr = attr_meta["name"] - is_minutes = attr_meta.get("minutes", False) - validators = group_validators(config_class, attr) - if config_instance: - raw_value = getattr(config_instance, attr, "") - if is_minutes and raw_value != "": - try: - value = str( - raw_value.total_seconds() / 60.0 - ) - except AttributeError: - value = str(raw_value) - else: - value = str(raw_value) - else: - value = "" - label = f"{attr.replace('_', ' ').title()}:" - yield Label( - label - if not is_minutes - else label.replace(":", " Minutes:") - ) - yield Input( - id=f"{instrument_name}_{attr}", - type=type_to_textual( - get_field_type(config_class, attr) - ), - validators=[ - Function( - validator, - f"INVALID: value must be {validator.__doc__.lower()}", - ) - for validator in validators - ], - value=value, - ) - yield Label( - "", - id=f"validation-failure-label-{instrument_name}_{attr}", - classes="-hidden validation-failure", - ) + else: + new_wp = Waypoint( + location=Location(latitude=0.0, longitude=0.0), + time=None, + instrument=[], + ) + self.expedition.schedule.waypoints.append(new_wp) + self.refresh_waypoint_widgets() + + except Exception as e: + raise UnexpectedError(unexpected_msg_compose(e)) from None + + @on(Button.Pressed, "#remove_waypoint") + def remove_waypoint(self) -> None: + """Remove the last waypoint from the schedule.""" + try: + if self.expedition.schedule.waypoints: + self.expedition.schedule.waypoints.pop() + self.refresh_waypoint_widgets() + else: + self.notify("No waypoints to remove.", severity="error", timeout=5) except Exception as e: raise UnexpectedError(unexpected_msg_compose(e)) from None @@ -885,31 +724,6 @@ def info_pressed(self) -> None: timeout=20, ) - @on(Input.Changed) - def show_invalid_reasons(self, event: Input.Changed) -> None: - input_id = event.input.id - label_id = f"validation-failure-label-{input_id}" - label = self.query_one(f"#{label_id}", Label) - if not event.validation_result.is_valid: - message = ( - "\n".join(event.validation_result.failure_descriptions) - if isinstance(event.validation_result.failure_descriptions, list) - else str(event.validation_result.failure_descriptions) - ) - label.update(message) - label.remove_class("-hidden") - label.add_class("validation-failure") - else: - label.update("") - label.add_class("-hidden") - label.remove_class("validation-failure") - - def on_mount(self) -> None: - adcp_present = ( - getattr(self.config, "adcp_config", None) if self.config else False - ) - self.show_hide_adcp_type(bool(adcp_present)) - def show_hide_adcp_type(self, show: bool) -> None: container = self.query_one("#adcp_type_container") if show: @@ -919,29 +733,32 @@ def show_hide_adcp_type(self, show: bool) -> None: def _set_adcp_default_values(self): self.query_one("#adcp_config_num_bins").value = str( - self.DEFAULT_ADCP_CONFIG["num_bins"] + DEFAULT_ADCP_CONFIG["num_bins"] ) self.query_one("#adcp_config_period").value = str( - self.DEFAULT_ADCP_CONFIG["period_minutes"] + DEFAULT_ADCP_CONFIG["period_minutes"] ) self.query_one("#adcp_shallow").value = False self.query_one("#adcp_deep").value = True def _set_ts_default_values(self): self.query_one("#ship_underwater_st_config_period").value = str( - self.DEFAULT_TS_CONFIG["period_minutes"] + DEFAULT_TS_CONFIG["period_minutes"] ) @on(Switch.Changed, "#has_adcp") def on_adcp_toggle(self, event: Switch.Changed) -> None: self.show_hide_adcp_type(event.value) - if event.value and not self.config.adcp_config: + if event.value and not self.expedition.instruments_config.adcp_config: # ADCP was turned on and was previously null self._set_adcp_default_values() @on(Switch.Changed, "#has_onboard_ts") def on_ts_toggle(self, event: Switch.Changed) -> None: - if event.value and not self.config.ship_underwater_st_config: + if ( + event.value + and not self.expedition.instruments_config.ship_underwater_st_config + ): # T/S was turned on and was previously null self._set_ts_default_values() @@ -957,68 +774,212 @@ def shallow_changed(self, event: Switch.Changed) -> None: deep = self.query_one("#adcp_deep", Switch) deep.value = False - def save_changes(self) -> bool: - """Save changes to ship_config.yaml.""" + +class WaypointWidget(Static): + def __init__(self, waypoint: Waypoint, index: int): + super().__init__() + self.waypoint = waypoint + self.index = index + + def compose(self) -> ComposeResult: try: - # ship speed - attr = "ship_speed_knots" - field_type = get_field_type(type(self.config), attr) - value = field_type(self.query_one("#speed").value) - ShipConfig.model_validate( - {**self.config.model_dump(), attr: value} - ) # validate using a temporary model (raises if invalid) - self.config.ship_speed_knots = value - - # individual instrument configurations - for instrument_name, info in self.INSTRUMENT_FIELDS.items(): - config_class = info["class"] - attributes = info["attributes"] - kwargs = {} - - # special handling for onboard ADCP and T/S - # will skip to next instrument if toggle is off - if instrument_name == "adcp_config": - has_adcp = self.query_one("#has_adcp", Switch).value - if not has_adcp: - setattr(self.config, instrument_name, None) - continue - if instrument_name == "ship_underwater_st_config": - has_ts = self.query_one("#has_onboard_ts", Switch).value - if not has_ts: - setattr(self.config, instrument_name, None) - continue - - for attr_meta in attributes: - attr = attr_meta["name"] - is_minutes = attr_meta.get("minutes", False) - input_id = f"{instrument_name}_{attr}" - value = self.query_one(f"#{input_id}").value - field_type = get_field_type(config_class, attr) - if is_minutes and field_type is datetime.timedelta: - value = datetime.timedelta(minutes=float(value)) - else: - value = field_type(value) - kwargs[attr] = value - - # ADCP max_depth_meter based on deep/shallow switch - if instrument_name == "adcp_config": - if self.query_one("#adcp_deep", Switch).value: - kwargs["max_depth_meter"] = -1000.0 - else: - kwargs["max_depth_meter"] = -150.0 - - setattr(self.config, instrument_name, config_class(**kwargs)) - - # save - self.config.to_yaml(f"{self.path}/ship_config.yaml") - return True + with Collapsible( + title=f"[b]Waypoint {self.index + 1}[/b]", + collapsed=True, + id=f"wp{self.index + 1}", + ): + if self.index > 0: + yield Button( + "Copy Time & Instruments from Previous", + id=f"wp{self.index}_copy", + variant="warning", + ) + yield Label("Location:") + yield Label(" Latitude:") + yield Input( + id=f"wp{self.index}_lat", + value=str(self.waypoint.location.lat) + if self.waypoint.location.lat + is not None # is not None to handle if lat is 0.0 + else "", + validators=[ + Function( + is_valid_lat, + f"INVALID: value must be {is_valid_lat.__doc__.lower()}", + ) + ], + type="number", + placeholder="°N", + classes="latitude-input", + ) + yield Label( + "", + id=f"validation-failure-label-wp{self.index}_lat", + classes="-hidden validation-failure", + ) + + yield Label(" Longitude:") + yield Input( + id=f"wp{self.index}_lon", + value=str(self.waypoint.location.lon) + if self.waypoint.location.lon + is not None # is not None to handle if lon is 0.0 + else "", + validators=[ + Function( + is_valid_lon, + f"INVALID: value must be {is_valid_lon.__doc__.lower()}", + ) + ], + type="number", + placeholder="°E", + classes="longitude-input", + ) + yield Label( + "", + id=f"validation-failure-label-wp{self.index}_lon", + classes="-hidden validation-failure", + ) + + yield Label("Time:") + with Horizontal(): + yield Label("Year:") + yield Select( + [ + (str(year), year) + # TODO: change from hard coding? ...flexibility for different datasets... + for year in range( + 2022, + datetime.datetime.now().year + 1, + ) + ], + id=f"wp{self.index}_year", + value=int(self.waypoint.time.year) + if self.waypoint.time + else Select.BLANK, + prompt="YYYY", + classes="year-select", + ) + yield Label("Month:") + yield Select( + [(f"{m:02d}", m) for m in range(1, 13)], + id=f"wp{self.index}_month", + value=int(self.waypoint.time.month) + if self.waypoint.time + else Select.BLANK, + prompt="MM", + classes="month-select", + ) + yield Label("Day:") + yield Select( + [(f"{d:02d}", d) for d in range(1, 32)], + id=f"wp{self.index}_day", + value=int(self.waypoint.time.day) + if self.waypoint.time + else Select.BLANK, + prompt="DD", + classes="day-select", + ) + yield Label("Hour:") + yield Select( + [(f"{h:02d}", h) for h in range(24)], + id=f"wp{self.index}_hour", + value=int(self.waypoint.time.hour) + if self.waypoint.time + else Select.BLANK, + prompt="hh", + classes="hour-select", + ) + yield Label("Min:") + yield Select( + [(f"{m:02d}", m) for m in range(0, 60, 5)], + id=f"wp{self.index}_minute", + value=int(self.waypoint.time.minute) + if self.waypoint.time + else Select.BLANK, + prompt="mm", + classes="minute-select", + ) + + yield Label("Instruments:") + for instrument in InstrumentType: + is_selected = instrument in (self.waypoint.instrument or []) + with Horizontal(): + yield Label(instrument.value) + yield Switch( + value=is_selected, id=f"wp{self.index}_{instrument.value}" + ) + + if instrument.value == "DRIFTER": + yield Label("Count") + yield Input( + id=f"wp{self.index}_drifter_count", + value=str( + self.get_drifter_count() if is_selected else "" + ), + type="integer", + placeholder="# of drifters", + validators=Integer( + minimum=1, + failure_description="INVALID: value must be > 0", + ), + classes="drifter-count-input", + ) + yield Label( + "", + id=f"validation-failure-label-wp{self.index}_drifter_count", + classes="-hidden validation-failure", + ) except Exception as e: - log_exception_to_file( - e, self.path, context_message="Error saving ship config:" - ) + raise UnexpectedError(unexpected_msg_compose(e)) from None + + def get_drifter_count(self) -> int: + return sum( + 1 for inst in self.waypoint.instrument if inst == InstrumentType.DRIFTER + ) + + def copy_from_previous(self) -> None: + """Copy inputs from previous waypoint widget (time and instruments only, not lat/lon).""" + try: + if self.index > 0: + schedule_editor = self.parent + if schedule_editor: + time_components = ["year", "month", "day", "hour", "minute"] + for comp in time_components: + prev = schedule_editor.query_one(f"#wp{self.index - 1}_{comp}") + curr = self.query_one(f"#wp{self.index}_{comp}") + if prev and curr: + curr.value = prev.value + + for instrument in InstrumentType: + prev_switch = schedule_editor.query_one( + f"#wp{self.index - 1}_{instrument.value}" + ) + curr_switch = self.query_one( + f"#wp{self.index}_{instrument.value}" + ) + if prev_switch and curr_switch: + curr_switch.value = prev_switch.value + except Exception as e: + raise UnexpectedError(unexpected_msg_compose(e)) from None + + @on(Button.Pressed, "Button") + def button_pressed(self, event: Button.Pressed) -> None: + if event.button.id == f"wp{self.index}_copy": + self.copy_from_previous() - raise UnexpectedError(UNEXPECTED_MSG_ONSAVE) from None + @on(Switch.Changed) + def on_switch_changed(self, event: Switch.Changed) -> None: + if event.switch.id == f"wp{self.index}_DRIFTER": + drifter_count_input = self.query_one( + f"#wp{self.index}_drifter_count", Input + ) + if not event.value: + drifter_count_input.value = "" + else: + if not drifter_count_input.value: + drifter_count_input.value = "1" class PlanScreen(Screen): @@ -1029,8 +990,7 @@ def __init__(self, path: str): def compose(self) -> ComposeResult: try: with VerticalScroll(): - yield ConfigEditor(self.path) - yield ScheduleEditor(self.path) + yield ExpeditionEditor(self.path) with Horizontal(): yield Button("Save Changes", id="save_button", variant="success") yield Button("Exit", id="exit_button", variant="error") @@ -1039,20 +999,20 @@ def compose(self) -> ComposeResult: def sync_ui_waypoints(self): """Update the waypoints models with current UI values (spacetime only) from the live UI inputs.""" - schedule_editor = self.query_one(ScheduleEditor) + expedition_editor = self.query_one(ExpeditionEditor) errors = [] - for i, wp in enumerate(schedule_editor.schedule.waypoints): + for i, wp in enumerate(expedition_editor.expedition.schedule.waypoints): try: wp.location = Location( - latitude=float(schedule_editor.query_one(f"#wp{i}_lat").value), - longitude=float(schedule_editor.query_one(f"#wp{i}_lon").value), + latitude=float(expedition_editor.query_one(f"#wp{i}_lat").value), + longitude=float(expedition_editor.query_one(f"#wp{i}_lon").value), ) wp.time = datetime.datetime( - int(schedule_editor.query_one(f"#wp{i}_year").value), - int(schedule_editor.query_one(f"#wp{i}_month").value), - int(schedule_editor.query_one(f"#wp{i}_day").value), - int(schedule_editor.query_one(f"#wp{i}_hour").value), - int(schedule_editor.query_one(f"#wp{i}_minute").value), + int(expedition_editor.query_one(f"#wp{i}_year").value), + int(expedition_editor.query_one(f"#wp{i}_month").value), + int(expedition_editor.query_one(f"#wp{i}_day").value), + int(expedition_editor.query_one(f"#wp{i}_hour").value), + int(expedition_editor.query_one(f"#wp{i}_minute").value), 0, ) except Exception as e: @@ -1075,26 +1035,24 @@ def exit_pressed(self) -> None: @on(Button.Pressed, "#save_button") def save_pressed(self) -> None: """Save button press.""" - config_editor = self.query_one(ConfigEditor) - schedule_editor = self.query_one(ScheduleEditor) + expedition_editor = self.query_one(ExpeditionEditor) try: - ship_speed_value = self.get_ship_speed(config_editor) + ship_speed_value = self.get_ship_speed(expedition_editor) self.sync_ui_waypoints() # call to ensure waypoint inputs are synced # verify schedule - schedule_editor.schedule.verify( + expedition_editor.expedition.schedule.verify( ship_speed_value, input_data=None, check_space_time_region=True, ignore_missing_fieldsets=True, ) - config_saved = config_editor.save_changes() - schedule_saved = schedule_editor.save_changes() + expedition_saved = expedition_editor.save_changes() - if config_saved and schedule_saved: + if expedition_saved: self.notify( "Changes saved successfully", severity="information", @@ -1109,9 +1067,9 @@ def save_pressed(self) -> None: ) return False - def get_ship_speed(self, config_editor): + def get_ship_speed(self, expedition_editor): try: - ship_speed = float(config_editor.query_one("#speed").value) + ship_speed = float(expedition_editor.query_one("#speed").value) assert ship_speed > 0 except Exception as e: log_exception_to_file( @@ -1130,12 +1088,6 @@ class PlanApp(App): align: center middle; } - ConfigEditor { - padding: 1; - margin-bottom: 1; - height: auto; - } - VerticalScroll { width: 100%; height: 100%; @@ -1210,7 +1162,12 @@ class PlanApp(App): margin: 0 1; } - #title { + #title_ship_instruments_config { + text-style: bold; + padding: 1; + } + + #title_schedule { text-style: bold; padding: 1; } diff --git a/src/virtualship/cli/commands.py b/src/virtualship/cli/commands.py index 72d378667..3e83be3b9 100644 --- a/src/virtualship/cli/commands.py +++ b/src/virtualship/cli/commands.py @@ -7,8 +7,7 @@ from virtualship.cli._plan import _plan from virtualship.expedition.do_expedition import do_expedition from virtualship.utils import ( - SCHEDULE, - SHIP_CONFIG, + EXPEDITION, mfp_to_yaml, ) @@ -28,47 +27,39 @@ ) def init(path, from_mfp): """ - Initialize a directory for a new expedition, with an example schedule and ship config files. + Initialize a directory for a new expedition, with an expedition.yaml file. - If --mfp-file is provided, it will generate the schedule from the MPF file instead. + If --mfp-file is provided, it will generate the expedition.yaml from the MPF file instead. """ path = Path(path) path.mkdir(exist_ok=True) - config = path / SHIP_CONFIG - schedule = path / SCHEDULE + expedition = path / EXPEDITION - if config.exists(): + if expedition.exists(): raise FileExistsError( - f"File '{config}' already exist. Please remove it or choose another directory." + f"File '{expedition}' already exist. Please remove it or choose another directory." ) - if schedule.exists(): - raise FileExistsError( - f"File '{schedule}' already exist. Please remove it or choose another directory." - ) - - config.write_text(utils.get_example_config()) if from_mfp: mfp_file = Path(from_mfp) - # Generate schedule.yaml from the MPF file + # Generate expedition.yaml from the MPF file click.echo(f"Generating schedule from {mfp_file}...") - mfp_to_yaml(mfp_file, schedule) + mfp_to_yaml(mfp_file, expedition) click.echo( "\n⚠️ The generated schedule does not contain TIME values or INSTRUMENT selections. ⚠️" "\n\nNow please either use the `\033[4mvirtualship plan\033[0m` app to complete the schedule configuration, " - "\nOR edit 'schedule.yaml' and manually add the necessary time values and instrument selections." - "\n\nIf editing 'schedule.yaml' manually:" + "\nOR edit 'expedition.yaml' and manually add the necessary time values and instrument selections under the 'schedule' heading." + "\n\nIf editing 'expedition.yaml' manually:" "\n\n🕒 Expected time format: 'YYYY-MM-DD HH:MM:SS' (e.g., '2023-10-20 01:00:00')." "\n\n🌡️ Expected instrument(s) format: one line per instrument e.g." f"\n\n{' ' * 15}waypoints:\n{' ' * 15}- instrument:\n{' ' * 19}- CTD\n{' ' * 19}- ARGO_FLOAT\n" ) else: - # Create a default example schedule - # schedule_body = utils.get_example_schedule() - schedule.write_text(utils.get_example_schedule()) + # Create a default example expedition YAML + expedition.write_text(utils.get_example_expedition()) - click.echo(f"Created '{config.name}' and '{schedule.name}' at {path}.") + click.echo(f"Created '{expedition.name}' at {path}.") @click.command() diff --git a/src/virtualship/expedition/do_expedition.py b/src/virtualship/expedition/do_expedition.py index 56ee79fa4..5c46d2eb8 100644 --- a/src/virtualship/expedition/do_expedition.py +++ b/src/virtualship/expedition/do_expedition.py @@ -7,11 +7,10 @@ import pyproj from virtualship.cli._fetch import get_existing_download, get_space_time_region_hash -from virtualship.models import Schedule, ShipConfig +from virtualship.models import Expedition, Schedule from virtualship.utils import ( CHECKPOINT, - _get_schedule, - _get_ship_config, + _get_expedition, ) from .checkpoint import Checkpoint @@ -38,11 +37,10 @@ def do_expedition(expedition_dir: str | Path, input_data: Path | None = None) -> if isinstance(expedition_dir, str): expedition_dir = Path(expedition_dir) - ship_config = _get_ship_config(expedition_dir) - schedule = _get_schedule(expedition_dir) + expedition = _get_expedition(expedition_dir) - # Verify ship_config file is consistent with schedule - ship_config.verify(schedule) + # Verify instruments_config file is consistent with schedule + expedition.instruments_config.verify(expedition.schedule) # load last checkpoint checkpoint = _load_checkpoint(expedition_dir) @@ -50,24 +48,26 @@ def do_expedition(expedition_dir: str | Path, input_data: Path | None = None) -> checkpoint = Checkpoint(past_schedule=Schedule(waypoints=[])) # verify that schedule and checkpoint match - checkpoint.verify(schedule) + checkpoint.verify(expedition.schedule) # load fieldsets loaded_input_data = _load_input_data( expedition_dir=expedition_dir, - schedule=schedule, - ship_config=ship_config, + expedition=expedition, input_data=input_data, ) print("\n---- WAYPOINT VERIFICATION ----") # verify schedule is valid - schedule.verify(ship_config.ship_speed_knots, loaded_input_data) + expedition.schedule.verify( + expedition.ship_config.ship_speed_knots, loaded_input_data + ) # simulate the schedule schedule_results = simulate_schedule( - projection=projection, ship_config=ship_config, schedule=schedule + projection=projection, + expedition=expedition, ) if isinstance(schedule_results, ScheduleProblem): print( @@ -76,7 +76,9 @@ def do_expedition(expedition_dir: str | Path, input_data: Path | None = None) -> _save_checkpoint( Checkpoint( past_schedule=Schedule( - waypoints=schedule.waypoints[: schedule_results.failed_waypoint_i] + waypoints=expedition.schedule.waypoints[ + : schedule_results.failed_waypoint_i + ] ) ), expedition_dir, @@ -91,10 +93,10 @@ def do_expedition(expedition_dir: str | Path, input_data: Path | None = None) -> print("\n----- EXPEDITION SUMMARY ------") # calculate expedition cost in US$ - assert schedule.waypoints[0].time is not None, ( + assert expedition.schedule.waypoints[0].time is not None, ( "First waypoint has no time. This should not be possible as it should have been verified before." ) - time_past = schedule_results.time - schedule.waypoints[0].time + time_past = schedule_results.time - expedition.schedule.waypoints[0].time cost = expedition_cost(schedule_results, time_past) with open(expedition_dir.joinpath("results", "cost.txt"), "w") as file: file.writelines(f"cost: {cost} US$") @@ -106,7 +108,7 @@ def do_expedition(expedition_dir: str | Path, input_data: Path | None = None) -> print("\nSimulating measurements. This may take a while...\n") simulate_measurements( expedition_dir, - ship_config, + expedition.instruments_config, loaded_input_data, schedule_results.measurements_to_simulate, ) @@ -122,26 +124,21 @@ def do_expedition(expedition_dir: str | Path, input_data: Path | None = None) -> def _load_input_data( expedition_dir: Path, - schedule: Schedule, - ship_config: ShipConfig, + expedition: Expedition, input_data: Path | None, ) -> InputData: """ Load the input data. :param expedition_dir: Directory of the expedition. - :type expedition_dir: Path - :param schedule: Schedule object. - :type schedule: Schedule - :param ship_config: Ship configuration. - :type ship_config: ShipConfig + :param expedition: Expedition object. :param input_data: Folder containing input data. - :type input_data: Path | None :return: InputData object. - :rtype: InputData """ if input_data is None: - space_time_region_hash = get_space_time_region_hash(schedule.space_time_region) + space_time_region_hash = get_space_time_region_hash( + expedition.schedule.space_time_region + ) input_data = get_existing_download(expedition_dir, space_time_region_hash) assert input_data is not None, ( @@ -150,13 +147,14 @@ def _load_input_data( return InputData.load( directory=input_data, - load_adcp=ship_config.adcp_config is not None, - load_argo_float=ship_config.argo_float_config is not None, - load_ctd=ship_config.ctd_config is not None, - load_ctd_bgc=ship_config.ctd_bgc_config is not None, - load_drifter=ship_config.drifter_config is not None, - load_xbt=ship_config.xbt_config is not None, - load_ship_underwater_st=ship_config.ship_underwater_st_config is not None, + load_adcp=expedition.instruments_config.adcp_config is not None, + load_argo_float=expedition.instruments_config.argo_float_config is not None, + load_ctd=expedition.instruments_config.ctd_config is not None, + load_ctd_bgc=expedition.instruments_config.ctd_bgc_config is not None, + load_drifter=expedition.instruments_config.drifter_config is not None, + load_xbt=expedition.instruments_config.xbt_config is not None, + load_ship_underwater_st=expedition.instruments_config.ship_underwater_st_config + is not None, ) diff --git a/src/virtualship/expedition/simulate_measurements.py b/src/virtualship/expedition/simulate_measurements.py index 20ba2cdbf..6cb2e4880 100644 --- a/src/virtualship/expedition/simulate_measurements.py +++ b/src/virtualship/expedition/simulate_measurements.py @@ -16,7 +16,7 @@ from virtualship.instruments.drifter import simulate_drifters from virtualship.instruments.ship_underwater_st import simulate_ship_underwater_st from virtualship.instruments.xbt import simulate_xbt -from virtualship.models import ShipConfig +from virtualship.models import InstrumentsConfig from virtualship.utils import ship_spinner from .simulate_schedule import MeasurementsToSimulate @@ -31,7 +31,7 @@ def simulate_measurements( expedition_dir: str | Path, - ship_config: ShipConfig, + instruments_config: InstrumentsConfig, input_data: InputData, measurements: MeasurementsToSimulate, ) -> None: @@ -41,7 +41,6 @@ def simulate_measurements( Saves everything in expedition_dir/results. :param expedition_dir: Base directory of the expedition. - :param ship_config: Ship configuration. :param input_data: Input data for simulation. :param measurements: The measurements to simulate. :raises RuntimeError: In case fieldsets of configuration is not provided. Make sure to check this before calling this function. @@ -50,7 +49,7 @@ def simulate_measurements( expedition_dir = Path(expedition_dir) if len(measurements.ship_underwater_sts) > 0: - if ship_config.ship_underwater_st_config is None: + if instruments_config.ship_underwater_st_config is None: raise RuntimeError("No configuration for ship underwater ST provided.") if input_data.ship_underwater_st_fieldset is None: raise RuntimeError("No fieldset for ship underwater ST provided.") @@ -68,7 +67,7 @@ def simulate_measurements( spinner.ok("✅") if len(measurements.adcps) > 0: - if ship_config.adcp_config is None: + if instruments_config.adcp_config is None: raise RuntimeError("No configuration for ADCP provided.") if input_data.adcp_fieldset is None: raise RuntimeError("No fieldset for ADCP provided.") @@ -78,15 +77,15 @@ def simulate_measurements( simulate_adcp( fieldset=input_data.adcp_fieldset, out_path=expedition_dir.joinpath("results", "adcp.zarr"), - max_depth=ship_config.adcp_config.max_depth_meter, + max_depth=instruments_config.adcp_config.max_depth_meter, min_depth=-5, - num_bins=ship_config.adcp_config.num_bins, + num_bins=instruments_config.adcp_config.num_bins, sample_points=measurements.adcps, ) spinner.ok("✅") if len(measurements.ctds) > 0: - if ship_config.ctd_config is None: + if instruments_config.ctd_config is None: raise RuntimeError("No configuration for CTD provided.") if input_data.ctd_fieldset is None: raise RuntimeError("No fieldset for CTD provided.") @@ -102,7 +101,7 @@ def simulate_measurements( spinner.ok("✅") if len(measurements.ctd_bgcs) > 0: - if ship_config.ctd_bgc_config is None: + if instruments_config.ctd_bgc_config is None: raise RuntimeError("No configuration for CTD_BGC provided.") if input_data.ctd_bgc_fieldset is None: raise RuntimeError("No fieldset for CTD_BGC provided.") @@ -118,7 +117,7 @@ def simulate_measurements( spinner.ok("✅") if len(measurements.xbts) > 0: - if ship_config.xbt_config is None: + if instruments_config.xbt_config is None: raise RuntimeError("No configuration for XBTs provided.") if input_data.xbt_fieldset is None: raise RuntimeError("No fieldset for XBTs provided.") @@ -135,7 +134,7 @@ def simulate_measurements( if len(measurements.drifters) > 0: print("Simulating drifters... ") - if ship_config.drifter_config is None: + if instruments_config.drifter_config is None: raise RuntimeError("No configuration for drifters provided.") if input_data.drifter_fieldset is None: raise RuntimeError("No fieldset for drifters provided.") @@ -150,7 +149,7 @@ def simulate_measurements( if len(measurements.argo_floats) > 0: print("Simulating argo floats... ") - if ship_config.argo_float_config is None: + if instruments_config.argo_float_config is None: raise RuntimeError("No configuration for argo floats provided.") if input_data.argo_float_fieldset is None: raise RuntimeError("No fieldset for argo floats provided.") diff --git a/src/virtualship/expedition/simulate_schedule.py b/src/virtualship/expedition/simulate_schedule.py index 95fa2f5fe..3b78c5c72 100644 --- a/src/virtualship/expedition/simulate_schedule.py +++ b/src/virtualship/expedition/simulate_schedule.py @@ -13,10 +13,9 @@ from virtualship.instruments.drifter import Drifter from virtualship.instruments.xbt import XBT from virtualship.models import ( + Expedition, InstrumentType, Location, - Schedule, - ShipConfig, Spacetime, Waypoint, ) @@ -52,23 +51,21 @@ class MeasurementsToSimulate: def simulate_schedule( - projection: pyproj.Geod, ship_config: ShipConfig, schedule: Schedule + projection: pyproj.Geod, expedition: Expedition ) -> ScheduleOk | ScheduleProblem: """ Simulate a schedule. :param projection: The projection to use for sailing. - :param ship_config: Ship configuration. - :param schedule: The schedule to simulate. + :param expedition: Expedition object containing the schedule to simulate. :returns: Either the results of a successfully simulated schedule, or information on where the schedule became infeasible. """ - return _ScheduleSimulator(projection, ship_config, schedule).simulate() + return _ScheduleSimulator(projection, expedition).simulate() class _ScheduleSimulator: _projection: pyproj.Geod - _ship_config: ShipConfig - _schedule: Schedule + _expedition: Expedition _time: datetime """Current time.""" @@ -82,18 +79,15 @@ class _ScheduleSimulator: _next_ship_underwater_st_time: datetime """Next moment ship underwater ST measurement will be done.""" - def __init__( - self, projection: pyproj.Geod, ship_config: ShipConfig, schedule: Schedule - ) -> None: + def __init__(self, projection: pyproj.Geod, expedition: Expedition) -> None: self._projection = projection - self._ship_config = ship_config - self._schedule = schedule + self._expedition = expedition - assert self._schedule.waypoints[0].time is not None, ( + assert self._expedition.schedule.waypoints[0].time is not None, ( "First waypoint must have a time. This should have been verified before calling this function." ) - self._time = schedule.waypoints[0].time - self._location = schedule.waypoints[0].location + self._time = expedition.schedule.waypoints[0].time + self._location = expedition.schedule.waypoints[0].location self._measurements_to_simulate = MeasurementsToSimulate() @@ -101,7 +95,7 @@ def __init__( self._next_ship_underwater_st_time = self._time def simulate(self) -> ScheduleOk | ScheduleProblem: - for wp_i, waypoint in enumerate(self._schedule.waypoints): + for wp_i, waypoint in enumerate(self._expedition.schedule.waypoints): # sail towards waypoint self._progress_time_traveling_towards(waypoint.location) @@ -131,7 +125,9 @@ def _progress_time_traveling_towards(self, location: Location) -> None: lons2=location.lon, lats2=location.lat, ) - ship_speed_meter_per_second = self._ship_config.ship_speed_knots * 1852 / 3600 + ship_speed_meter_per_second = ( + self._expedition.ship_config.ship_speed_knots * 1852 / 3600 + ) azimuth1 = geodinv[0] distance_to_next_waypoint = geodinv[2] time_to_reach = timedelta( @@ -140,7 +136,7 @@ def _progress_time_traveling_towards(self, location: Location) -> None: end_time = self._time + time_to_reach # note all ADCP measurements - if self._ship_config.adcp_config is not None: + if self._expedition.instruments_config.adcp_config is not None: location = self._location time = self._time while self._next_adcp_time <= end_time: @@ -162,11 +158,12 @@ def _progress_time_traveling_towards(self, location: Location) -> None: ) self._next_adcp_time = ( - self._next_adcp_time + self._ship_config.adcp_config.period + self._next_adcp_time + + self._expedition.instruments_config.adcp_config.period ) # note all ship underwater ST measurements - if self._ship_config.ship_underwater_st_config is not None: + if self._expedition.instruments_config.ship_underwater_st_config is not None: location = self._location time = self._time while self._next_ship_underwater_st_time <= end_time: @@ -189,7 +186,7 @@ def _progress_time_traveling_towards(self, location: Location) -> None: self._next_ship_underwater_st_time = ( self._next_ship_underwater_st_time - + self._ship_config.ship_underwater_st_config.period + + self._expedition.instruments_config.ship_underwater_st_config.period ) self._time = end_time @@ -199,24 +196,25 @@ def _progress_time_stationary(self, time_passed: timedelta) -> None: end_time = self._time + time_passed # note all ADCP measurements - if self._ship_config.adcp_config is not None: + if self._expedition.instruments_config.adcp_config is not None: while self._next_adcp_time <= end_time: self._measurements_to_simulate.adcps.append( Spacetime(self._location, self._next_adcp_time) ) self._next_adcp_time = ( - self._next_adcp_time + self._ship_config.adcp_config.period + self._next_adcp_time + + self._expedition.instruments_config.adcp_config.period ) # note all ship underwater ST measurements - if self._ship_config.ship_underwater_st_config is not None: + if self._expedition.instruments_config.ship_underwater_st_config is not None: while self._next_ship_underwater_st_time <= end_time: self._measurements_to_simulate.ship_underwater_sts.append( Spacetime(self._location, self._next_ship_underwater_st_time) ) self._next_ship_underwater_st_time = ( self._next_ship_underwater_st_time - + self._ship_config.ship_underwater_st_config.period + + self._expedition.instruments_config.ship_underwater_st_config.period ) self._time = end_time @@ -241,48 +239,52 @@ def _make_measurements(self, waypoint: Waypoint) -> timedelta: self._measurements_to_simulate.argo_floats.append( ArgoFloat( spacetime=Spacetime(self._location, self._time), - min_depth=self._ship_config.argo_float_config.min_depth_meter, - max_depth=self._ship_config.argo_float_config.max_depth_meter, - drift_depth=self._ship_config.argo_float_config.drift_depth_meter, - vertical_speed=self._ship_config.argo_float_config.vertical_speed_meter_per_second, - cycle_days=self._ship_config.argo_float_config.cycle_days, - drift_days=self._ship_config.argo_float_config.drift_days, + min_depth=self._expedition.instruments_config.argo_float_config.min_depth_meter, + max_depth=self._expedition.instruments_config.argo_float_config.max_depth_meter, + drift_depth=self._expedition.instruments_config.argo_float_config.drift_depth_meter, + vertical_speed=self._expedition.instruments_config.argo_float_config.vertical_speed_meter_per_second, + cycle_days=self._expedition.instruments_config.argo_float_config.cycle_days, + drift_days=self._expedition.instruments_config.argo_float_config.drift_days, ) ) elif instrument is InstrumentType.CTD: self._measurements_to_simulate.ctds.append( CTD( spacetime=Spacetime(self._location, self._time), - min_depth=self._ship_config.ctd_config.min_depth_meter, - max_depth=self._ship_config.ctd_config.max_depth_meter, + min_depth=self._expedition.instruments_config.ctd_config.min_depth_meter, + max_depth=self._expedition.instruments_config.ctd_config.max_depth_meter, ) ) - time_costs.append(self._ship_config.ctd_config.stationkeeping_time) + time_costs.append( + self._expedition.instruments_config.ctd_config.stationkeeping_time + ) elif instrument is InstrumentType.CTD_BGC: self._measurements_to_simulate.ctd_bgcs.append( CTD_BGC( spacetime=Spacetime(self._location, self._time), - min_depth=self._ship_config.ctd_bgc_config.min_depth_meter, - max_depth=self._ship_config.ctd_bgc_config.max_depth_meter, + min_depth=self._expedition.instruments_config.ctd_bgc_config.min_depth_meter, + max_depth=self._expedition.instruments_config.ctd_bgc_config.max_depth_meter, ) ) - time_costs.append(self._ship_config.ctd_bgc_config.stationkeeping_time) + time_costs.append( + self._expedition.instruments_config.ctd_bgc_config.stationkeeping_time + ) elif instrument is InstrumentType.DRIFTER: self._measurements_to_simulate.drifters.append( Drifter( spacetime=Spacetime(self._location, self._time), - depth=self._ship_config.drifter_config.depth_meter, - lifetime=self._ship_config.drifter_config.lifetime, + depth=self._expedition.instruments_config.drifter_config.depth_meter, + lifetime=self._expedition.instruments_config.drifter_config.lifetime, ) ) elif instrument is InstrumentType.XBT: self._measurements_to_simulate.xbts.append( XBT( spacetime=Spacetime(self._location, self._time), - min_depth=self._ship_config.xbt_config.min_depth_meter, - max_depth=self._ship_config.xbt_config.max_depth_meter, - fall_speed=self._ship_config.xbt_config.fall_speed_meter_per_second, - deceleration_coefficient=self._ship_config.xbt_config.deceleration_coefficient, + min_depth=self._expedition.instruments_config.xbt_config.min_depth_meter, + max_depth=self._expedition.instruments_config.xbt_config.max_depth_meter, + fall_speed=self._expedition.instruments_config.xbt_config.fall_speed_meter_per_second, + deceleration_coefficient=self._expedition.instruments_config.xbt_config.deceleration_coefficient, ) ) else: diff --git a/src/virtualship/models/__init__.py b/src/virtualship/models/__init__.py index 481060564..a2f1546cb 100644 --- a/src/virtualship/models/__init__.py +++ b/src/virtualship/models/__init__.py @@ -1,18 +1,21 @@ """Pydantic models and data classes used to configure virtualship (i.e., in the configuration files or settings).""" -from .location import Location -from .schedule import Schedule, Waypoint -from .ship_config import ( +from .expedition import ( ADCPConfig, ArgoFloatConfig, CTD_BGCConfig, CTDConfig, DrifterConfig, + Expedition, + InstrumentsConfig, InstrumentType, + Schedule, ShipConfig, ShipUnderwaterSTConfig, + Waypoint, XBTConfig, ) +from .location import Location from .space_time_region import ( SpaceTimeRegion, SpatialRange, @@ -25,6 +28,7 @@ __all__ = [ # noqa: RUF022 "Location", "Schedule", + "ShipConfig", "Waypoint", "InstrumentType", "ArgoFloatConfig", @@ -34,9 +38,10 @@ "ShipUnderwaterSTConfig", "DrifterConfig", "XBTConfig", - "ShipConfig", "SpatialRange", "TimeRange", "SpaceTimeRegion", "Spacetime", + "Expedition", + "InstrumentsConfig", ] diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py new file mode 100644 index 000000000..2e073b842 --- /dev/null +++ b/src/virtualship/models/expedition.py @@ -0,0 +1,456 @@ +from __future__ import annotations + +import itertools +from datetime import datetime, timedelta +from enum import Enum +from typing import TYPE_CHECKING + +import pydantic +import pyproj +import yaml + +from virtualship.errors import ConfigError, ScheduleError +from virtualship.utils import _validate_numeric_mins_to_timedelta + +from .location import Location +from .space_time_region import SpaceTimeRegion + +if TYPE_CHECKING: + from parcels import FieldSet + + from virtualship.expedition.input_data import InputData + + +projection: pyproj.Geod = pyproj.Geod(ellps="WGS84") + + +class Expedition(pydantic.BaseModel): + """Expedition class, including schedule and ship config.""" + + schedule: Schedule + instruments_config: InstrumentsConfig + ship_config: ShipConfig + + model_config = pydantic.ConfigDict(extra="forbid") + + def to_yaml(self, file_path: str) -> None: + """Write exepedition object to yaml file.""" + with open(file_path, "w") as file: + yaml.dump(self.model_dump(by_alias=True), file) + + @classmethod + def from_yaml(cls, file_path: str) -> Expedition: + """Load config from yaml file.""" + with open(file_path) as file: + data = yaml.safe_load(file) + return Expedition(**data) + + +class ShipConfig(pydantic.BaseModel): + """Configuration of the ship.""" + + ship_speed_knots: float = pydantic.Field(gt=0.0) + + # TODO: room here for adding more ship config options in future PRs (e.g. max_days_at_sea)... + + model_config = pydantic.ConfigDict(extra="forbid") + + +class Schedule(pydantic.BaseModel): + """Schedule of the virtual ship.""" + + waypoints: list[Waypoint] + space_time_region: SpaceTimeRegion | None = None + + model_config = pydantic.ConfigDict(extra="forbid") + + def get_instruments(self) -> set[InstrumentType]: + """Return a set of unique InstrumentType enums used in the schedule.""" + instruments_in_schedule = [] + for waypoint in self.waypoints: + if waypoint.instrument: + for instrument in waypoint.instrument: + if instrument: + instruments_in_schedule.append(instrument) + return set(instruments_in_schedule) + + def verify( + self, + ship_speed: float, + input_data: InputData | None, + *, + check_space_time_region: bool = False, + ignore_missing_fieldsets: bool = False, + ) -> None: + """ + Verify the feasibility and correctness of the schedule's waypoints. + + This method checks various conditions to ensure the schedule is valid: + 1. At least one waypoint is provided. + 2. The first waypoint has a specified time. + 3. Waypoint times are in ascending order. + 4. All waypoints are in water (not on land). + 5. The ship can arrive on time at each waypoint given its speed. + + :param ship_speed: The ship's speed in knots. + :param input_data: An InputData object containing fieldsets used to check if waypoints are on water. + :param check_space_time_region: whether to check for missing space_time_region. + :param ignore_missing_fieldsets: whether to ignore warning for missing field sets. + :raises PlanningError: If any of the verification checks fail, indicating infeasible or incorrect waypoints. + :raises NotImplementedError: If an instrument in the schedule is not implemented. + :return: None. The method doesn't return a value but raises exceptions if verification fails. + """ + print("\nVerifying route... ") + + if check_space_time_region and self.space_time_region is None: + raise ScheduleError( + "space_time_region not found in schedule, please define it to fetch the data." + ) + + if len(self.waypoints) == 0: + raise ScheduleError("At least one waypoint must be provided.") + + # check first waypoint has a time + if self.waypoints[0].time is None: + raise ScheduleError("First waypoint must have a specified time.") + + # check waypoint times are in ascending order + timed_waypoints = [wp for wp in self.waypoints if wp.time is not None] + checks = [ + next.time >= cur.time for cur, next in itertools.pairwise(timed_waypoints) + ] + if not all(checks): + invalid_i = [i for i, c in enumerate(checks) if c] + raise ScheduleError( + f"Waypoint(s) {', '.join(f'#{i + 1}' for i in invalid_i)}: each waypoint should be timed after all previous waypoints", + ) + + # check if all waypoints are in water + # this is done by picking an arbitrary provided fieldset and checking if UV is not zero + + # get all available fieldsets + available_fieldsets = [] + if input_data is not None: + fieldsets = [ + input_data.adcp_fieldset, + input_data.argo_float_fieldset, + input_data.ctd_fieldset, + input_data.drifter_fieldset, + input_data.ship_underwater_st_fieldset, + ] + for fs in fieldsets: + if fs is not None: + available_fieldsets.append(fs) + + # check if there are any fieldsets, else it's an error + if len(available_fieldsets) == 0: + if not ignore_missing_fieldsets: + print( + "Cannot verify because no fieldsets have been loaded. This is probably " + "because you are not using any instruments in your schedule. This is not a problem, " + "but carefully check your waypoint locations manually." + ) + + else: + # pick any + fieldset = available_fieldsets[0] + # get waypoints with 0 UV + land_waypoints = [ + (wp_i, wp) + for wp_i, wp in enumerate(self.waypoints) + if _is_on_land_zero_uv(fieldset, wp) + ] + # raise an error if there are any + if len(land_waypoints) > 0: + raise ScheduleError( + f"The following waypoints are on land: {['#' + str(wp_i) + ' ' + str(wp) for (wp_i, wp) in land_waypoints]}" + ) + + # check that ship will arrive on time at each waypoint (in case no unexpected event happen) + time = self.waypoints[0].time + for wp_i, (wp, wp_next) in enumerate( + zip(self.waypoints, self.waypoints[1:], strict=False) + ): + if wp.instrument is InstrumentType.CTD: + time += timedelta(minutes=20) + + geodinv: tuple[float, float, float] = projection.inv( + wp.location.lon, + wp.location.lat, + wp_next.location.lon, + wp_next.location.lat, + ) + distance = geodinv[2] + + time_to_reach = timedelta(seconds=distance / ship_speed * 3600 / 1852) + arrival_time = time + time_to_reach + + if wp_next.time is None: + time = arrival_time + elif arrival_time > wp_next.time: + raise ScheduleError( + f"Waypoint planning is not valid: would arrive too late at waypoint number {wp_i + 2}. " + f"location: {wp_next.location} time: {wp_next.time} instrument: {wp_next.instrument}" + ) + else: + time = wp_next.time + + print("... All good to go!") + + +class Waypoint(pydantic.BaseModel): + """A Waypoint to sail to with an optional time and an optional instrument.""" + + location: Location + time: datetime | None = None + instrument: InstrumentType | list[InstrumentType] | None = None + + @pydantic.field_serializer("instrument") + def serialize_instrument(self, instrument): + """Ensure InstrumentType is serialized as a string (or list of strings).""" + if isinstance(instrument, list): + return [inst.value for inst in instrument] + return instrument.value if instrument else None + + +class InstrumentType(Enum): + """Types of the instruments.""" + + CTD = "CTD" + CTD_BGC = "CTD_BGC" + DRIFTER = "DRIFTER" + ARGO_FLOAT = "ARGO_FLOAT" + XBT = "XBT" + + +class ArgoFloatConfig(pydantic.BaseModel): + """Configuration for argos floats.""" + + min_depth_meter: float = pydantic.Field(le=0.0) + max_depth_meter: float = pydantic.Field(le=0.0) + drift_depth_meter: float = pydantic.Field(le=0.0) + vertical_speed_meter_per_second: float = pydantic.Field(lt=0.0) + cycle_days: float = pydantic.Field(gt=0.0) + drift_days: float = pydantic.Field(gt=0.0) + + +class ADCPConfig(pydantic.BaseModel): + """Configuration for ADCP instrument.""" + + max_depth_meter: float = pydantic.Field(le=0.0) + num_bins: int = pydantic.Field(gt=0.0) + period: timedelta = pydantic.Field( + serialization_alias="period_minutes", + validation_alias="period_minutes", + gt=timedelta(), + ) + + model_config = pydantic.ConfigDict(populate_by_name=True) + + @pydantic.field_serializer("period") + def _serialize_period(self, value: timedelta, _info): + return value.total_seconds() / 60.0 + + @pydantic.field_validator("period", mode="before") + def _validate_period(cls, value: int | float | timedelta) -> timedelta: + return _validate_numeric_mins_to_timedelta(value) + + +class CTDConfig(pydantic.BaseModel): + """Configuration for CTD instrument.""" + + stationkeeping_time: timedelta = pydantic.Field( + serialization_alias="stationkeeping_time_minutes", + validation_alias="stationkeeping_time_minutes", + gt=timedelta(), + ) + min_depth_meter: float = pydantic.Field(le=0.0) + max_depth_meter: float = pydantic.Field(le=0.0) + + model_config = pydantic.ConfigDict(populate_by_name=True) + + @pydantic.field_serializer("stationkeeping_time") + def _serialize_stationkeeping_time(self, value: timedelta, _info): + return value.total_seconds() / 60.0 + + @pydantic.field_validator("stationkeeping_time", mode="before") + def _validate_stationkeeping_time(cls, value: int | float | timedelta) -> timedelta: + return _validate_numeric_mins_to_timedelta(value) + + +class CTD_BGCConfig(pydantic.BaseModel): + """Configuration for CTD_BGC instrument.""" + + stationkeeping_time: timedelta = pydantic.Field( + serialization_alias="stationkeeping_time_minutes", + validation_alias="stationkeeping_time_minutes", + gt=timedelta(), + ) + min_depth_meter: float = pydantic.Field(le=0.0) + max_depth_meter: float = pydantic.Field(le=0.0) + + model_config = pydantic.ConfigDict(populate_by_name=True) + + @pydantic.field_serializer("stationkeeping_time") + def _serialize_stationkeeping_time(self, value: timedelta, _info): + return value.total_seconds() / 60.0 + + @pydantic.field_validator("stationkeeping_time", mode="before") + def _validate_stationkeeping_time(cls, value: int | float | timedelta) -> timedelta: + return _validate_numeric_mins_to_timedelta(value) + + +class ShipUnderwaterSTConfig(pydantic.BaseModel): + """Configuration for underwater ST.""" + + period: timedelta = pydantic.Field( + serialization_alias="period_minutes", + validation_alias="period_minutes", + gt=timedelta(), + ) + + model_config = pydantic.ConfigDict(populate_by_name=True) + + @pydantic.field_serializer("period") + def _serialize_period(self, value: timedelta, _info): + return value.total_seconds() / 60.0 + + @pydantic.field_validator("period", mode="before") + def _validate_period(cls, value: int | float | timedelta) -> timedelta: + return _validate_numeric_mins_to_timedelta(value) + + +class DrifterConfig(pydantic.BaseModel): + """Configuration for drifters.""" + + depth_meter: float = pydantic.Field(le=0.0) + lifetime: timedelta = pydantic.Field( + serialization_alias="lifetime_minutes", + validation_alias="lifetime_minutes", + gt=timedelta(), + ) + + model_config = pydantic.ConfigDict(populate_by_name=True) + + @pydantic.field_serializer("lifetime") + def _serialize_lifetime(self, value: timedelta, _info): + return value.total_seconds() / 60.0 + + @pydantic.field_validator("lifetime", mode="before") + def _validate_lifetime(cls, value: int | float | timedelta) -> timedelta: + return _validate_numeric_mins_to_timedelta(value) + + +class XBTConfig(pydantic.BaseModel): + """Configuration for xbt instrument.""" + + min_depth_meter: float = pydantic.Field(le=0.0) + max_depth_meter: float = pydantic.Field(le=0.0) + fall_speed_meter_per_second: float = pydantic.Field(gt=0.0) + deceleration_coefficient: float = pydantic.Field(gt=0.0) + + +class InstrumentsConfig(pydantic.BaseModel): + """Configuration of instruments.""" + + argo_float_config: ArgoFloatConfig | None = None + """ + Argo float configuration. + + If None, no argo floats can be deployed. + """ + + adcp_config: ADCPConfig | None = None + """ + ADCP configuration. + + If None, no ADCP measurements will be performed. + """ + + ctd_config: CTDConfig | None = None + """ + CTD configuration. + + If None, no CTDs can be cast. + """ + + ctd_bgc_config: CTD_BGCConfig | None = None + """ + CTD_BGC configuration. + + If None, no BGC CTDs can be cast. + """ + + ship_underwater_st_config: ShipUnderwaterSTConfig | None = None + """ + Ship underwater salinity temperature measurementconfiguration. + + If None, no ST measurements will be performed. + """ + + drifter_config: DrifterConfig | None = None + """ + Drifter configuration. + + If None, no drifters can be deployed. + """ + + xbt_config: XBTConfig | None = None + """ + XBT configuration. + + If None, no XBTs can be cast. + """ + + model_config = pydantic.ConfigDict(extra="forbid") + + def verify(self, schedule: Schedule) -> None: + """ + Verify instrument configurations against the schedule. + + Removes instrument configs not present in the schedule and checks that all scheduled instruments are configured. + Raises ConfigError if any scheduled instrument is missing a config. + """ + instruments_in_schedule = schedule.get_instruments() + instrument_config_map = { + InstrumentType.ARGO_FLOAT: "argo_float_config", + InstrumentType.DRIFTER: "drifter_config", + InstrumentType.XBT: "xbt_config", + InstrumentType.CTD: "ctd_config", + InstrumentType.CTD_BGC: "ctd_bgc_config", + } + # Remove configs for unused instruments + for inst_type, config_attr in instrument_config_map.items(): + if hasattr(self, config_attr) and inst_type not in instruments_in_schedule: + print( + f"{inst_type.value} configuration provided but not in schedule. Removing config." + ) + setattr(self, config_attr, None) + # Check all scheduled instruments are configured + for inst_type in instruments_in_schedule: + config_attr = instrument_config_map.get(inst_type) + if ( + not config_attr + or not hasattr(self, config_attr) + or getattr(self, config_attr) is None + ): + raise ConfigError( + f"Schedule includes instrument '{inst_type.value}', but instruments_config does not provide configuration for it." + ) + + +def _is_on_land_zero_uv(fieldset: FieldSet, waypoint: Waypoint) -> bool: + """ + Check if waypoint is on land by assuming zero velocity means land. + + :param fieldset: The fieldset to sample the velocity from. + :param waypoint: The waypoint to check. + :returns: If the waypoint is on land. + """ + return fieldset.UV.eval( + 0, + fieldset.gridset.grids[0].depth[0], + waypoint.location.lat, + waypoint.location.lon, + applyConversion=False, + ) == (0.0, 0.0) diff --git a/src/virtualship/models/schedule.py b/src/virtualship/models/schedule.py deleted file mode 100644 index 3de44f093..000000000 --- a/src/virtualship/models/schedule.py +++ /dev/null @@ -1,236 +0,0 @@ -"""Schedule class.""" - -from __future__ import annotations - -import itertools -from datetime import datetime, timedelta -from pathlib import Path -from typing import TYPE_CHECKING - -import pydantic -import pyproj -import yaml - -from virtualship.errors import ScheduleError - -from .location import Location -from .ship_config import InstrumentType -from .space_time_region import SpaceTimeRegion - -if TYPE_CHECKING: - from parcels import FieldSet - - from virtualship.expedition.input_data import InputData - -projection: pyproj.Geod = pyproj.Geod(ellps="WGS84") - - -class Waypoint(pydantic.BaseModel): - """A Waypoint to sail to with an optional time and an optional instrument.""" - - location: Location - time: datetime | None = None - instrument: InstrumentType | list[InstrumentType] | None = None - - @pydantic.field_serializer("instrument") - def serialize_instrument(self, instrument): - """Ensure InstrumentType is serialized as a string (or list of strings).""" - if isinstance(instrument, list): - return [inst.value for inst in instrument] - return instrument.value if instrument else None - - -class Schedule(pydantic.BaseModel): - """Schedule of the virtual ship.""" - - waypoints: list[Waypoint] - space_time_region: SpaceTimeRegion | None = None - - model_config = pydantic.ConfigDict(extra="forbid") - - def to_yaml(self, file_path: str | Path) -> None: - """ - Write schedule to yaml file. - - :param file_path: Path to the file to write to. - """ - with open(file_path, "w") as file: - yaml.dump( - self.model_dump( - by_alias=True, - ), - file, - ) - - @classmethod - def from_yaml(cls, file_path: str | Path) -> Schedule: - """ - Load schedule from yaml file. - - :param file_path: Path to the file to load from. - :returns: The schedule. - """ - with open(file_path) as file: - data = yaml.safe_load(file) - return Schedule(**data) - - def get_instruments(self) -> set[InstrumentType]: - """ - Retrieve a set of unique instruments used in the schedule. - - This method iterates through all waypoints in the schedule and collects - the instruments associated with each waypoint. It returns a set of unique - instruments, either as objects or as names. - - :raises CheckpointError: If the past waypoints in the given schedule - have been changed compared to the checkpoint. - :return: set: A set of unique instruments used in the schedule. - - """ - instruments_in_schedule = [] - for waypoint in self.waypoints: - if waypoint.instrument: - for instrument in waypoint.instrument: - if instrument: - instruments_in_schedule.append(instrument) - return set(instruments_in_schedule) - - def verify( - self, - ship_speed: float, - input_data: InputData | None, - *, - check_space_time_region: bool = False, - ignore_missing_fieldsets: bool = False, - ) -> None: - """ - Verify the feasibility and correctness of the schedule's waypoints. - - This method checks various conditions to ensure the schedule is valid: - 1. At least one waypoint is provided. - 2. The first waypoint has a specified time. - 3. Waypoint times are in ascending order. - 4. All waypoints are in water (not on land). - 5. The ship can arrive on time at each waypoint given its speed. - - :param ship_speed: The ship's speed in knots. - :param input_data: An InputData object containing fieldsets used to check if waypoints are on water. - :param check_space_time_region: whether to check for missing space_time_region. - :param ignore_missing_fieldsets: whether to ignore warning for missing field sets. - :raises PlanningError: If any of the verification checks fail, indicating infeasible or incorrect waypoints. - :raises NotImplementedError: If an instrument in the schedule is not implemented. - :return: None. The method doesn't return a value but raises exceptions if verification fails. - """ - print("\nVerifying route... ") - - if check_space_time_region and self.space_time_region is None: - raise ScheduleError( - "space_time_region not found in schedule, please define it to fetch the data." - ) - - if len(self.waypoints) == 0: - raise ScheduleError("At least one waypoint must be provided.") - - # check first waypoint has a time - if self.waypoints[0].time is None: - raise ScheduleError("First waypoint must have a specified time.") - - # check waypoint times are in ascending order - timed_waypoints = [wp for wp in self.waypoints if wp.time is not None] - checks = [ - next.time >= cur.time for cur, next in itertools.pairwise(timed_waypoints) - ] - if not all(checks): - invalid_i = [i for i, c in enumerate(checks) if c] - raise ScheduleError( - f"Waypoint(s) {', '.join(f'#{i + 1}' for i in invalid_i)}: each waypoint should be timed after all previous waypoints", - ) - - # check if all waypoints are in water - # this is done by picking an arbitrary provided fieldset and checking if UV is not zero - - # get all available fieldsets - available_fieldsets = [] - if input_data is not None: - fieldsets = [ - input_data.adcp_fieldset, - input_data.argo_float_fieldset, - input_data.ctd_fieldset, - input_data.drifter_fieldset, - input_data.ship_underwater_st_fieldset, - ] - for fs in fieldsets: - if fs is not None: - available_fieldsets.append(fs) - - # check if there are any fieldsets, else it's an error - if len(available_fieldsets) == 0: - if not ignore_missing_fieldsets: - print( - "Cannot verify because no fieldsets have been loaded. This is probably " - "because you are not using any instruments in your schedule. This is not a problem, " - "but carefully check your waypoint locations manually." - ) - - else: - # pick any - fieldset = available_fieldsets[0] - # get waypoints with 0 UV - land_waypoints = [ - (wp_i, wp) - for wp_i, wp in enumerate(self.waypoints) - if _is_on_land_zero_uv(fieldset, wp) - ] - # raise an error if there are any - if len(land_waypoints) > 0: - raise ScheduleError( - f"The following waypoints are on land: {['#' + str(wp_i) + ' ' + str(wp) for (wp_i, wp) in land_waypoints]}" - ) - - # check that ship will arrive on time at each waypoint (in case no unexpected event happen) - time = self.waypoints[0].time - for wp_i, (wp, wp_next) in enumerate( - zip(self.waypoints, self.waypoints[1:], strict=False) - ): - if wp.instrument is InstrumentType.CTD: - time += timedelta(minutes=20) - - geodinv: tuple[float, float, float] = projection.inv( - wp.location.lon, - wp.location.lat, - wp_next.location.lon, - wp_next.location.lat, - ) - distance = geodinv[2] - - time_to_reach = timedelta(seconds=distance / ship_speed * 3600 / 1852) - arrival_time = time + time_to_reach - - if wp_next.time is None: - time = arrival_time - elif arrival_time > wp_next.time: - raise ScheduleError( - f"Waypoint planning is not valid: would arrive too late at waypoint number {wp_i + 2}. " - f"location: {wp_next.location} time: {wp_next.time} instrument: {wp_next.instrument}" - ) - else: - time = wp_next.time - - print("... All good to go!") - - -def _is_on_land_zero_uv(fieldset: FieldSet, waypoint: Waypoint) -> bool: - """ - Check if waypoint is on land by assuming zero velocity means land. - - :param fieldset: The fieldset to sample the velocity from. - :param waypoint: The waypoint to check. - :returns: If the waypoint is on land. - """ - return fieldset.UV.eval( - 0, - fieldset.gridset.grids[0].depth[0], - waypoint.location.lat, - waypoint.location.lon, - applyConversion=False, - ) == (0.0, 0.0) diff --git a/src/virtualship/models/ship_config.py b/src/virtualship/models/ship_config.py deleted file mode 100644 index be3ee30dc..000000000 --- a/src/virtualship/models/ship_config.py +++ /dev/null @@ -1,320 +0,0 @@ -"""ShipConfig and supporting classes.""" - -from __future__ import annotations - -from datetime import timedelta -from enum import Enum -from pathlib import Path -from typing import TYPE_CHECKING - -import pydantic -import yaml - -from virtualship.errors import ConfigError -from virtualship.utils import _validate_numeric_mins_to_timedelta - -if TYPE_CHECKING: - from .schedule import Schedule - - -class InstrumentType(Enum): - """Types of the instruments.""" - - CTD = "CTD" - CTD_BGC = "CTD_BGC" - DRIFTER = "DRIFTER" - ARGO_FLOAT = "ARGO_FLOAT" - XBT = "XBT" - - -class ArgoFloatConfig(pydantic.BaseModel): - """Configuration for argos floats.""" - - min_depth_meter: float = pydantic.Field(le=0.0) - max_depth_meter: float = pydantic.Field(le=0.0) - drift_depth_meter: float = pydantic.Field(le=0.0) - vertical_speed_meter_per_second: float = pydantic.Field(lt=0.0) - cycle_days: float = pydantic.Field(gt=0.0) - drift_days: float = pydantic.Field(gt=0.0) - - -class ADCPConfig(pydantic.BaseModel): - """Configuration for ADCP instrument.""" - - max_depth_meter: float = pydantic.Field(le=0.0) - num_bins: int = pydantic.Field(gt=0.0) - period: timedelta = pydantic.Field( - serialization_alias="period_minutes", - validation_alias="period_minutes", - gt=timedelta(), - ) - - model_config = pydantic.ConfigDict(populate_by_name=True) - - @pydantic.field_serializer("period") - def _serialize_period(self, value: timedelta, _info): - return value.total_seconds() / 60.0 - - @pydantic.field_validator("period", mode="before") - def _validate_period(cls, value: int | float | timedelta) -> timedelta: - return _validate_numeric_mins_to_timedelta(value) - - -class CTDConfig(pydantic.BaseModel): - """Configuration for CTD instrument.""" - - stationkeeping_time: timedelta = pydantic.Field( - serialization_alias="stationkeeping_time_minutes", - validation_alias="stationkeeping_time_minutes", - gt=timedelta(), - ) - min_depth_meter: float = pydantic.Field(le=0.0) - max_depth_meter: float = pydantic.Field(le=0.0) - - model_config = pydantic.ConfigDict(populate_by_name=True) - - @pydantic.field_serializer("stationkeeping_time") - def _serialize_stationkeeping_time(self, value: timedelta, _info): - return value.total_seconds() / 60.0 - - @pydantic.field_validator("stationkeeping_time", mode="before") - def _validate_stationkeeping_time(cls, value: int | float | timedelta) -> timedelta: - return _validate_numeric_mins_to_timedelta(value) - - -class CTD_BGCConfig(pydantic.BaseModel): - """Configuration for CTD_BGC instrument.""" - - stationkeeping_time: timedelta = pydantic.Field( - serialization_alias="stationkeeping_time_minutes", - validation_alias="stationkeeping_time_minutes", - gt=timedelta(), - ) - min_depth_meter: float = pydantic.Field(le=0.0) - max_depth_meter: float = pydantic.Field(le=0.0) - - model_config = pydantic.ConfigDict(populate_by_name=True) - - @pydantic.field_serializer("stationkeeping_time") - def _serialize_stationkeeping_time(self, value: timedelta, _info): - return value.total_seconds() / 60.0 - - @pydantic.field_validator("stationkeeping_time", mode="before") - def _validate_stationkeeping_time(cls, value: int | float | timedelta) -> timedelta: - return _validate_numeric_mins_to_timedelta(value) - - -class ShipUnderwaterSTConfig(pydantic.BaseModel): - """Configuration for underwater ST.""" - - period: timedelta = pydantic.Field( - serialization_alias="period_minutes", - validation_alias="period_minutes", - gt=timedelta(), - ) - - model_config = pydantic.ConfigDict(populate_by_name=True) - - @pydantic.field_serializer("period") - def _serialize_period(self, value: timedelta, _info): - return value.total_seconds() / 60.0 - - @pydantic.field_validator("period", mode="before") - def _validate_period(cls, value: int | float | timedelta) -> timedelta: - return _validate_numeric_mins_to_timedelta(value) - - -class DrifterConfig(pydantic.BaseModel): - """Configuration for drifters.""" - - depth_meter: float = pydantic.Field(le=0.0) - lifetime: timedelta = pydantic.Field( - serialization_alias="lifetime_minutes", - validation_alias="lifetime_minutes", - gt=timedelta(), - ) - - model_config = pydantic.ConfigDict(populate_by_name=True) - - @pydantic.field_serializer("lifetime") - def _serialize_lifetime(self, value: timedelta, _info): - return value.total_seconds() / 60.0 - - @pydantic.field_validator("lifetime", mode="before") - def _validate_lifetime(cls, value: int | float | timedelta) -> timedelta: - return _validate_numeric_mins_to_timedelta(value) - - -class XBTConfig(pydantic.BaseModel): - """Configuration for xbt instrument.""" - - min_depth_meter: float = pydantic.Field(le=0.0) - max_depth_meter: float = pydantic.Field(le=0.0) - fall_speed_meter_per_second: float = pydantic.Field(gt=0.0) - deceleration_coefficient: float = pydantic.Field(gt=0.0) - - -class ShipConfig(pydantic.BaseModel): - """Configuration of the virtual ship.""" - - ship_speed_knots: float = pydantic.Field(gt=0.0) - """ - Velocity of the ship in knots. - """ - - argo_float_config: ArgoFloatConfig | None = None - """ - Argo float configuration. - - If None, no argo floats can be deployed. - """ - - adcp_config: ADCPConfig | None = None - """ - ADCP configuration. - - If None, no ADCP measurements will be performed. - """ - - ctd_config: CTDConfig | None = None - """ - CTD configuration. - - If None, no CTDs can be cast. - """ - - ctd_bgc_config: CTD_BGCConfig | None = None - """ - CTD_BGC configuration. - - If None, no BGC CTDs can be cast. - """ - - ship_underwater_st_config: ShipUnderwaterSTConfig | None = None - """ - Ship underwater salinity temperature measurementconfiguration. - - If None, no ST measurements will be performed. - """ - - drifter_config: DrifterConfig | None = None - """ - Drifter configuration. - - If None, no drifters can be deployed. - """ - - xbt_config: XBTConfig | None = None - """ - XBT configuration. - - If None, no XBTs can be cast. - """ - - model_config = pydantic.ConfigDict(extra="forbid") - - def to_yaml(self, file_path: str | Path) -> None: - """ - Write config to yaml file. - - :param file_path: Path to the file to write to. - """ - with open(file_path, "w") as file: - yaml.dump(self.model_dump(by_alias=True), file) - - @classmethod - def from_yaml(cls, file_path: str | Path) -> ShipConfig: - """ - Load config from yaml file. - - :param file_path: Path to the file to load from. - :returns: The config. - """ - with open(file_path) as file: - data = yaml.safe_load(file) - return ShipConfig(**data) - - def verify(self, schedule: Schedule) -> None: - """ - Verify the ship configuration against the provided schedule. - - This function performs two main tasks: - 1. Removes instrument configurations that are not present in the schedule. - 2. Verifies that all instruments in the schedule have corresponding configurations. - - Parameters - ---------- - schedule : Schedule - The schedule object containing the planned instruments and waypoints. - - Returns - ------- - None - - Raises - ------ - ConfigError - If an instrument in the schedule does not have a corresponding configuration. - - Notes - ----- - - Prints a message if a configuration is provided for an instrument not in the schedule. - - Sets the configuration to None for instruments not in the schedule. - - Raises a ConfigError for each instrument in the schedule that lacks a configuration. - - """ - instruments_in_schedule = schedule.get_instruments() - - for instrument in [ - "ARGO_FLOAT", - "DRIFTER", - "XBT", - "CTD", - "CTD_BGC", - ]: # TODO make instrument names consistent capitals or lowercase throughout codebase - if hasattr(self, instrument.lower() + "_config") and not any( - instrument == schedule_instrument.name - for schedule_instrument in instruments_in_schedule - ): - print(f"{instrument} configuration provided but not in schedule.") - setattr(self, instrument.lower() + "_config", None) - - # verify instruments in schedule have configuration - # TODO: the ConfigError message could be improved to explain that the **schedule** file has X instrument but the **ship_config** file does not - for instrument in instruments_in_schedule: - try: - InstrumentType(instrument) - except ValueError as e: - raise NotImplementedError("Instrument not supported.") from e - - if instrument == InstrumentType.ARGO_FLOAT and ( - not hasattr(self, "argo_float_config") or self.argo_float_config is None - ): - raise ConfigError( - "Planning has a waypoint with Argo float instrument, but configuration does not configure Argo floats." - ) - if instrument == InstrumentType.CTD and ( - not hasattr(self, "ctd_config") or self.ctd_config is None - ): - raise ConfigError( - "Planning has a waypoint with CTD instrument, but configuration does not configure CTDs." - ) - if instrument == InstrumentType.CTD_BGC and ( - not hasattr(self, "ctd_bgc_config") or self.ctd_bgc_config is None - ): - raise ConfigError( - "Planning has a waypoint with CTD_BGC instrument, but configuration does not configure CTD_BGCs." - ) - if instrument == InstrumentType.DRIFTER and ( - not hasattr(self, "drifter_config") or self.drifter_config is None - ): - raise ConfigError( - "Planning has a waypoint with drifter instrument, but configuration does not configure drifters." - ) - - if instrument == InstrumentType.XBT and ( - not hasattr(self, "xbt_config") or self.xbt_config is None - ): - raise ConfigError( - "Planning has a waypoint with XBT instrument, but configuration does not configure XBT." - ) diff --git a/src/virtualship/static/expedition.yaml b/src/virtualship/static/expedition.yaml new file mode 100644 index 000000000..1a9e39223 --- /dev/null +++ b/src/virtualship/static/expedition.yaml @@ -0,0 +1,75 @@ +schedule: + space_time_region: + spatial_range: + minimum_longitude: -5 + maximum_longitude: 5 + minimum_latitude: -5 + maximum_latitude: 5 + minimum_depth: 0 + maximum_depth: 2000 + time_range: + start_time: 2023-01-01 00:00:00 + end_time: 2023-02-01 00:00:00 + waypoints: + - instrument: + - CTD + - CTD_BGC + location: + latitude: 0 + longitude: 0 + time: 2023-01-01 00:00:00 + - instrument: + - DRIFTER + - CTD + location: + latitude: 0.01 + longitude: 0.01 + time: 2023-01-01 01:00:00 + - instrument: + - ARGO_FLOAT + location: + latitude: 0.02 + longitude: 0.02 + time: 2023-01-01 02:00:00 + - instrument: + - XBT + location: + latitude: 0.03 + longitude: 0.03 + time: 2023-01-01 03:00:00 + - location: + latitude: 0.03 + longitude: 0.03 + time: 2023-01-01 03:00:00 +instruments_config: + adcp_config: + num_bins: 40 + max_depth_meter: -1000.0 + period_minutes: 5.0 + argo_float_config: + cycle_days: 10.0 + drift_days: 9.0 + drift_depth_meter: -1000.0 + max_depth_meter: -2000.0 + min_depth_meter: 0.0 + vertical_speed_meter_per_second: -0.1 + ctd_config: + max_depth_meter: -2000.0 + min_depth_meter: -11.0 + stationkeeping_time_minutes: 20.0 + ctd_bgc_config: + max_depth_meter: -2000.0 + min_depth_meter: -11.0 + stationkeeping_time_minutes: 20.0 + drifter_config: + depth_meter: 0.0 + lifetime_minutes: 60480.0 + xbt_config: + max_depth_meter: -285.0 + min_depth_meter: -2.0 + fall_speed_meter_per_second: 6.7 + deceleration_coefficient: 0.00225 + ship_underwater_st_config: + period_minutes: 5.0 +ship_config: + ship_speed_knots: 10.0 diff --git a/src/virtualship/static/schedule.yaml b/src/virtualship/static/schedule.yaml deleted file mode 100644 index 7cb394233..000000000 --- a/src/virtualship/static/schedule.yaml +++ /dev/null @@ -1,42 +0,0 @@ -space_time_region: - spatial_range: - minimum_longitude: -5 - maximum_longitude: 5 - minimum_latitude: -5 - maximum_latitude: 5 - minimum_depth: 0 - maximum_depth: 2000 - time_range: - start_time: 2023-01-01 00:00:00 - end_time: 2023-02-01 00:00:00 -waypoints: - - instrument: - - CTD - - CTD_BGC - location: - latitude: 0 - longitude: 0 - time: 2023-01-01 00:00:00 - - instrument: - - DRIFTER - - CTD - location: - latitude: 0.01 - longitude: 0.01 - time: 2023-01-01 01:00:00 - - instrument: - - ARGO_FLOAT - location: - latitude: 0.02 - longitude: 0.02 - time: 2023-01-01 02:00:00 - - instrument: - - XBT - location: - latitude: 0.03 - longitude: 0.03 - time: 2023-01-01 03:00:00 - - location: - latitude: 0.03 - longitude: 0.03 - time: 2023-01-01 03:00:00 diff --git a/src/virtualship/static/ship_config.yaml b/src/virtualship/static/ship_config.yaml deleted file mode 100644 index 34d6c6eae..000000000 --- a/src/virtualship/static/ship_config.yaml +++ /dev/null @@ -1,30 +0,0 @@ -ship_speed_knots: 10.0 -adcp_config: - num_bins: 40 - max_depth_meter: -1000.0 - period_minutes: 5.0 -argo_float_config: - cycle_days: 10.0 - drift_days: 9.0 - drift_depth_meter: -1000.0 - max_depth_meter: -2000.0 - min_depth_meter: 0.0 - vertical_speed_meter_per_second: -0.1 -ctd_config: - max_depth_meter: -2000.0 - min_depth_meter: -11.0 - stationkeeping_time_minutes: 20.0 -ctd_bgc_config: - max_depth_meter: -2000.0 - min_depth_meter: -11.0 - stationkeeping_time_minutes: 20.0 -drifter_config: - depth_meter: 0.0 - lifetime_minutes: 60480.0 -xbt_config: - max_depth_meter: -285.0 - min_depth_meter: -2.0 - fall_speed_meter_per_second: 6.7 - deceleration_coefficient: 0.00225 -ship_underwater_st_config: - period_minutes: 5.0 diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index 1f334f06b..0a39d0356 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -8,17 +8,15 @@ from pathlib import Path from typing import TYPE_CHECKING, TextIO -from yaspin import Spinner - if TYPE_CHECKING: - from virtualship.models import Schedule, ShipConfig + from virtualship.models import Expedition import pandas as pd import yaml from pydantic import BaseModel +from yaspin import Spinner -SCHEDULE = "schedule.yaml" -SHIP_CONFIG = "ship_config.yaml" +EXPEDITION = "expedition.yaml" CHECKPOINT = "checkpoint.yaml" @@ -28,15 +26,10 @@ def load_static_file(name: str) -> str: @lru_cache(None) -def get_example_config() -> str: - """Get the example configuration file.""" - return load_static_file(SHIP_CONFIG) - - @lru_cache(None) -def get_example_schedule() -> str: - """Get the example schedule file.""" - return load_static_file(SCHEDULE) +def get_example_expedition() -> str: + """Get the example unified expedition configuration file.""" + return load_static_file(EXPEDITION) def _dump_yaml(model: BaseModel, stream: TextIO) -> str | None: @@ -121,7 +114,7 @@ def validate_coordinates(coordinates_data): def mfp_to_yaml(coordinates_file_path: str, yaml_output_path: str): # noqa: D417 """ - Generates a YAML file with spatial and temporal information based on instrument data from MFP excel file. + Generates an expedition.yaml file with schedule information based on data from MFP excel file. The ship and instrument configurations entries in the YAML file are sourced from the static version. Parameters ---------- @@ -134,7 +127,10 @@ def mfp_to_yaml(coordinates_file_path: str, yaml_output_path: str): # noqa: D41 4. returns the yaml information. """ + # avoid circular imports from virtualship.models import ( + Expedition, + InstrumentsConfig, Location, Schedule, SpaceTimeRegion, @@ -188,8 +184,23 @@ def mfp_to_yaml(coordinates_file_path: str, yaml_output_path: str): # noqa: D41 space_time_region=space_time_region, ) + # extract instruments config from static + instruments_config = InstrumentsConfig.model_validate( + yaml.safe_load(get_example_expedition()).get("instruments_config") + ) + + # extract ship config from static + ship_config = yaml.safe_load(get_example_expedition()).get("ship_config") + + # combine to Expedition object + expedition = Expedition( + schedule=schedule, + instruments_config=instruments_config, + ship_config=ship_config, + ) + # Save to YAML file - schedule.to_yaml(yaml_output_path) + expedition.to_yaml(yaml_output_path) def _validate_numeric_mins_to_timedelta(value: int | float | timedelta) -> timedelta: @@ -199,26 +210,16 @@ def _validate_numeric_mins_to_timedelta(value: int | float | timedelta) -> timed return timedelta(minutes=value) -def _get_schedule(expedition_dir: Path) -> Schedule: - """Load Schedule object from yaml config file in `expedition_dir`.""" - from virtualship.models import Schedule - - file_path = expedition_dir.joinpath(SCHEDULE) - try: - return Schedule.from_yaml(file_path) - except FileNotFoundError as e: - raise FileNotFoundError(f'Schedule not found. Save it to "{file_path}".') from e - - -def _get_ship_config(expedition_dir: Path) -> ShipConfig: - from virtualship.models import ShipConfig +def _get_expedition(expedition_dir: Path) -> Expedition: + """Load Expedition object from yaml config file in `expedition_dir`.""" + from virtualship.models import Expedition - file_path = expedition_dir.joinpath(SHIP_CONFIG) + file_path = expedition_dir.joinpath(EXPEDITION) try: - return ShipConfig.from_yaml(file_path) + return Expedition.from_yaml(file_path) except FileNotFoundError as e: raise FileNotFoundError( - f'Ship config not found. Save it to "{file_path}".' + f'{EXPEDITION} not found. Save it to "{file_path}".' ) from e diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 015c32672..b8e797b76 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -4,7 +4,7 @@ from click.testing import CliRunner from virtualship.cli.commands import fetch, init -from virtualship.utils import SCHEDULE, SHIP_CONFIG +from virtualship.utils import EXPEDITION @pytest.fixture @@ -32,29 +32,16 @@ def test_init(): with runner.isolated_filesystem(): result = runner.invoke(init, ["."]) assert result.exit_code == 0 - config = Path(SHIP_CONFIG) - schedule = Path(SCHEDULE) + expedition = Path(EXPEDITION) - assert config.exists() - assert schedule.exists() + assert expedition.exists() -def test_init_existing_config(): +def test_init_existing_expedition(): runner = CliRunner() with runner.isolated_filesystem(): - config = Path(SHIP_CONFIG) - config.write_text("test") - - with pytest.raises(FileExistsError): - result = runner.invoke(init, ["."]) - raise result.exception - - -def test_init_existing_schedule(): - runner = CliRunner() - with runner.isolated_filesystem(): - schedule = Path(SCHEDULE) - schedule.write_text("test") + expedition = Path(EXPEDITION) + expedition.write_text("test") with pytest.raises(FileExistsError): result = runner.invoke(init, ["."]) diff --git a/tests/cli/test_fetch.py b/tests/cli/test_fetch.py index 856b72f69..693907337 100644 --- a/tests/cli/test_fetch.py +++ b/tests/cli/test_fetch.py @@ -16,8 +16,8 @@ hash_model, hash_to_filename, ) -from virtualship.models import Schedule, ShipConfig -from virtualship.utils import get_example_config, get_example_schedule +from virtualship.models import Expedition +from virtualship.utils import EXPEDITION, get_example_expedition @pytest.fixture @@ -32,31 +32,19 @@ def fake_download(output_filename, output_directory, **_): @pytest.fixture -def schedule(tmpdir): - out_path = tmpdir.join("schedule.yaml") +def expedition(tmpdir): + out_path = tmpdir.join(EXPEDITION) with open(out_path, "w") as file: - file.write(get_example_schedule()) + file.write(get_example_expedition()) - schedule = Schedule.from_yaml(out_path) + expedition = Expedition.from_yaml(out_path) - return schedule - - -@pytest.fixture -def ship_config(tmpdir): - out_path = tmpdir.join("ship_config.yaml") - - with open(out_path, "w") as file: - file.write(get_example_config()) - - ship_config = ShipConfig.from_yaml(out_path) - - return ship_config + return expedition @pytest.mark.usefixtures("copernicus_subset_no_download") -def test_fetch(schedule, ship_config, tmpdir): +def test_fetch(expedition, tmpdir): """Test the fetch command, but mock the download.""" _fetch(Path(tmpdir), "test", "test") diff --git a/tests/cli/test_plan.py b/tests/cli/test_plan.py index 6fef90a18..421feba0f 100644 --- a/tests/cli/test_plan.py +++ b/tests/cli/test_plan.py @@ -9,7 +9,8 @@ import yaml from textual.widgets import Button, Collapsible, Input -from virtualship.cli._plan import ConfigEditor, PlanApp, ScheduleEditor +from virtualship.cli._plan import ExpeditionEditor, PlanApp +from virtualship.utils import EXPEDITION NEW_SPEED = "8.0" NEW_LAT = "0.05" @@ -33,12 +34,8 @@ async def test_UI_changes(): tmpdir = Path(tempfile.mkdtemp()) shutil.copy( - files("virtualship.static").joinpath("ship_config.yaml"), - tmpdir / "ship_config.yaml", - ) - shutil.copy( - files("virtualship.static").joinpath("schedule.yaml"), - tmpdir / "schedule.yaml", + files("virtualship.static").joinpath(EXPEDITION), + tmpdir / EXPEDITION, ) app = PlanApp(path=tmpdir) @@ -47,22 +44,23 @@ async def test_UI_changes(): await pilot.pause(0.5) plan_screen = pilot.app.screen - config_editor = plan_screen.query_one(ConfigEditor) - schedule_editor = plan_screen.query_one(ScheduleEditor) + expedition_editor = plan_screen.query_one(ExpeditionEditor) # get mock of UI notify method plan_screen.notify = MagicMock() # change ship speed - speed_collapsible = config_editor.query_one("#speed_collapsible", Collapsible) + speed_collapsible = expedition_editor.query_one( + "#speed_collapsible", Collapsible + ) if speed_collapsible.collapsed: speed_collapsible.collapsed = False await pilot.pause() - ship_speed_input = config_editor.query_one("#speed", Input) + ship_speed_input = expedition_editor.query_one("#speed", Input) await simulate_input(pilot, ship_speed_input, NEW_SPEED) # change waypoint lat/lon (e.g. first waypoint) - waypoints_collapsible = schedule_editor.query_one("#waypoints", Collapsible) + waypoints_collapsible = expedition_editor.query_one("#waypoints", Collapsible) if waypoints_collapsible.collapsed: waypoints_collapsible.collapsed = False await pilot.pause() @@ -104,11 +102,11 @@ async def test_UI_changes(): ) # verify changes to speed, lat, lon in saved YAML - ship_config_path = os.path.join(tmpdir, "ship_config.yaml") - with open(ship_config_path) as f: - saved_config = yaml.safe_load(f) + expedition_path = os.path.join(tmpdir, EXPEDITION) + with open(expedition_path) as f: + saved_expedition = yaml.safe_load(f) - assert saved_config["ship_speed_knots"] == float(NEW_SPEED) + assert saved_expedition["ship_config"]["ship_speed_knots"] == float(NEW_SPEED) # check schedule.verify() methods are working by purposefully making invalid schedule (i.e. ship speed too slow to reach waypoints) invalid_speed = "0.0001" diff --git a/tests/expedition/expedition_dir/expedition.yaml b/tests/expedition/expedition_dir/expedition.yaml new file mode 100644 index 000000000..9468028f8 --- /dev/null +++ b/tests/expedition/expedition_dir/expedition.yaml @@ -0,0 +1,46 @@ +schedule: + waypoints: + - instrument: + - CTD + location: + latitude: 0 + longitude: 0 + time: 2023-01-01 00:00:00 + - instrument: + - DRIFTER + - ARGO_FLOAT + location: + latitude: 0.01 + longitude: 0.01 + time: 2023-01-02 00:00:00 + - location: # empty waypoint + latitude: 0.02 + longitude: 0.01 + time: 2023-01-02 03:00:00 +instruments_config: + adcp_config: + num_bins: 40 + max_depth_meter: -1000.0 + period_minutes: 5.0 + argo_float_config: + cycle_days: 10.0 + drift_days: 9.0 + drift_depth_meter: -1000.0 + max_depth_meter: -2000.0 + min_depth_meter: 0.0 + vertical_speed_meter_per_second: -0.1 + ctd_config: + max_depth_meter: -2000.0 + min_depth_meter: -11.0 + stationkeeping_time_minutes: 20.0 + ctd_bgc_config: + max_depth_meter: -2000.0 + min_depth_meter: -11.0 + stationkeeping_time_minutes: 20.0 + drifter_config: + depth_meter: 0.0 + lifetime_minutes: 40320.0 + ship_underwater_st_config: + period_minutes: 5.0 +ship_config: + ship_speed_knots: 10.0 diff --git a/tests/expedition/expedition_dir/schedule.yaml b/tests/expedition/expedition_dir/schedule.yaml deleted file mode 100644 index 29c14ac9b..000000000 --- a/tests/expedition/expedition_dir/schedule.yaml +++ /dev/null @@ -1,18 +0,0 @@ -waypoints: - - instrument: - - CTD - location: - latitude: 0 - longitude: 0 - time: 2023-01-01 00:00:00 - - instrument: - - DRIFTER - - ARGO_FLOAT - location: - latitude: 0.01 - longitude: 0.01 - time: 2023-01-02 00:00:00 - - location: # empty waypoint - latitude: 0.02 - longitude: 0.01 - time: 2023-01-02 03:00:00 diff --git a/tests/expedition/expedition_dir/ship_config.yaml b/tests/expedition/expedition_dir/ship_config.yaml deleted file mode 100644 index 1bae9d1dc..000000000 --- a/tests/expedition/expedition_dir/ship_config.yaml +++ /dev/null @@ -1,25 +0,0 @@ -ship_speed_knots: 10.0 -adcp_config: - num_bins: 40 - max_depth_meter: -1000.0 - period_minutes: 5.0 -argo_float_config: - cycle_days: 10.0 - drift_days: 9.0 - drift_depth_meter: -1000.0 - max_depth_meter: -2000.0 - min_depth_meter: 0.0 - vertical_speed_meter_per_second: -0.1 -ctd_config: - max_depth_meter: -2000.0 - min_depth_meter: -11.0 - stationkeeping_time_minutes: 20.0 -ctd_bgc_config: - max_depth_meter: -2000.0 - min_depth_meter: -11.0 - stationkeeping_time_minutes: 20.0 -drifter_config: - depth_meter: 0.0 - lifetime_minutes: 40320.0 -ship_underwater_st_config: - period_minutes: 5.0 diff --git a/tests/expedition/test_expedition.py b/tests/expedition/test_expedition.py new file mode 100644 index 000000000..a4643e03a --- /dev/null +++ b/tests/expedition/test_expedition.py @@ -0,0 +1,277 @@ +from datetime import datetime, timedelta +from pathlib import Path + +import pyproj +import pytest + +from virtualship.errors import ConfigError, ScheduleError +from virtualship.expedition.do_expedition import _load_input_data +from virtualship.models import Expedition, Location, Schedule, Waypoint +from virtualship.utils import EXPEDITION, _get_expedition, get_example_expedition + +projection = pyproj.Geod(ellps="WGS84") + +expedition_dir = Path("expedition_dir") + + +def test_import_export_expedition(tmpdir) -> None: + out_path = tmpdir.join(EXPEDITION) + + # arbitrary time for testing + base_time = datetime.strptime("1950-01-01", "%Y-%m-%d") + + schedule = Schedule( + waypoints=[ + Waypoint(location=Location(0, 0), time=base_time, instrument=None), + Waypoint( + location=Location(1, 1), + time=base_time + timedelta(hours=1), + instrument=None, + ), + ] + ) + get_expedition = _get_expedition(expedition_dir) + expedition = Expedition( + schedule=schedule, + instruments_config=get_expedition.instruments_config, + ship_config=get_expedition.ship_config, + ) + expedition.to_yaml(out_path) + + expedition2 = Expedition.from_yaml(out_path) + assert expedition == expedition2 + + +def test_verify_schedule() -> None: + schedule = Schedule( + waypoints=[ + Waypoint(location=Location(0, 0), time=datetime(2022, 1, 1, 1, 0, 0)), + Waypoint(location=Location(1, 0), time=datetime(2022, 1, 2, 1, 0, 0)), + ] + ) + + ship_speed_knots = _get_expedition(expedition_dir).ship_config.ship_speed_knots + + schedule.verify(ship_speed_knots, None) + + +def test_get_instruments() -> None: + schedule = Schedule( + waypoints=[ + Waypoint(location=Location(0, 0), instrument=["CTD"]), + Waypoint(location=Location(1, 0), instrument=["XBT", "ARGO_FLOAT"]), + Waypoint(location=Location(1, 0), instrument=["CTD"]), + ] + ) + + assert set(instrument.name for instrument in schedule.get_instruments()) == { + "CTD", + "XBT", + "ARGO_FLOAT", + } + + +@pytest.mark.parametrize( + "schedule,check_space_time_region,error,match", + [ + pytest.param( + Schedule(waypoints=[]), + False, + ScheduleError, + "At least one waypoint must be provided.", + id="NoWaypoints", + ), + pytest.param( + Schedule( + waypoints=[ + Waypoint(location=Location(0, 0)), + Waypoint( + location=Location(1, 0), time=datetime(2022, 1, 1, 1, 0, 0) + ), + ] + ), + False, + ScheduleError, + "First waypoint must have a specified time.", + id="FirstWaypointHasTime", + ), + pytest.param( + Schedule( + waypoints=[ + Waypoint( + location=Location(0, 0), time=datetime(2022, 1, 2, 1, 0, 0) + ), + Waypoint(location=Location(0, 0)), + Waypoint( + location=Location(1, 0), time=datetime(2022, 1, 1, 1, 0, 0) + ), + ] + ), + False, + ScheduleError, + "Waypoint\\(s\\) : each waypoint should be timed after all previous waypoints", + id="SequentialWaypoints", + ), + pytest.param( + Schedule( + waypoints=[ + Waypoint( + location=Location(0, 0), time=datetime(2022, 1, 1, 1, 0, 0) + ), + Waypoint( + location=Location(1, 0), time=datetime(2022, 1, 1, 1, 1, 0) + ), + ] + ), + False, + ScheduleError, + "Waypoint planning is not valid: would arrive too late at waypoint number 2...", + id="NotEnoughTime", + ), + pytest.param( + Schedule( + waypoints=[ + Waypoint( + location=Location(0, 0), time=datetime(2022, 1, 1, 1, 0, 0) + ), + Waypoint( + location=Location(1, 0), time=datetime(2022, 1, 2, 1, 1, 0) + ), + ] + ), + True, + ScheduleError, + "space_time_region not found in schedule, please define it to fetch the data.", + id="NoSpaceTimeRegion", + ), + ], +) +def test_verify_schedule_errors( + schedule: Schedule, check_space_time_region: bool, error, match +) -> None: + expedition = _get_expedition(expedition_dir) + input_data = _load_input_data( + expedition_dir, + expedition, + input_data=Path("expedition_dir/input_data"), + ) + + with pytest.raises(error, match=match): + schedule.verify( + expedition.ship_config.ship_speed_knots, + input_data, + check_space_time_region=check_space_time_region, + ) + + +@pytest.fixture +def schedule(tmp_file): + with open(tmp_file, "w") as file: + file.write(get_example_expedition()) + return Expedition.from_yaml(tmp_file).schedule + + +@pytest.fixture +def schedule_no_xbt(schedule): + for waypoint in schedule.waypoints: + if waypoint.instrument and any( + instrument.name == "XBT" for instrument in waypoint.instrument + ): + waypoint.instrument = [ + instrument + for instrument in waypoint.instrument + if instrument.name != "XBT" + ] + + return schedule + + +@pytest.fixture +def instruments_config(tmp_file): + with open(tmp_file, "w") as file: + file.write(get_example_expedition()) + return Expedition.from_yaml(tmp_file).instruments_config + + +@pytest.fixture +def instruments_config_no_xbt(instruments_config): + delattr(instruments_config, "xbt_config") + return instruments_config + + +@pytest.fixture +def instruments_config_no_ctd(instruments_config): + delattr(instruments_config, "ctd_config") + return instruments_config + + +@pytest.fixture +def instruments_config_no_ctd_bgc(instruments_config): + delattr(instruments_config, "ctd_bgc_config") + return instruments_config + + +@pytest.fixture +def instruments_config_no_argo_float(instruments_config): + delattr(instruments_config, "argo_float_config") + return instruments_config + + +@pytest.fixture +def instruments_config_no_drifter(instruments_config): + delattr(instruments_config, "drifter_config") + return instruments_config + + +def test_verify_instruments_config(instruments_config, schedule) -> None: + instruments_config.verify(schedule) + + +def test_verify_instruments_config_no_instrument( + instruments_config, schedule_no_xbt +) -> None: + instruments_config.verify(schedule_no_xbt) + + +@pytest.mark.parametrize( + "instruments_config_fixture,error,match", + [ + pytest.param( + "instruments_config_no_xbt", + ConfigError, + "Schedule includes instrument 'XBT', but instruments_config does not provide configuration for it.", + id="ShipConfigNoXBT", + ), + pytest.param( + "instruments_config_no_ctd", + ConfigError, + "Schedule includes instrument 'CTD', but instruments_config does not provide configuration for it.", + id="ShipConfigNoCTD", + ), + pytest.param( + "instruments_config_no_ctd_bgc", + ConfigError, + "Schedule includes instrument 'CTD_BGC', but instruments_config does not provide configuration for it.", + id="ShipConfigNoCTD_BGC", + ), + pytest.param( + "instruments_config_no_argo_float", + ConfigError, + "Schedule includes instrument 'ARGO_FLOAT', but instruments_config does not provide configuration for it.", + id="ShipConfigNoARGO_FLOAT", + ), + pytest.param( + "instruments_config_no_drifter", + ConfigError, + "Schedule includes instrument 'DRIFTER', but instruments_config does not provide configuration for it.", + id="ShipConfigNoDRIFTER", + ), + ], +) +def test_verify_instruments_config_errors( + request, schedule, instruments_config_fixture, error, match +) -> None: + instruments_config = request.getfixturevalue(instruments_config_fixture) + + with pytest.raises(error, match=match): + instruments_config.verify(schedule) diff --git a/tests/expedition/test_schedule.py b/tests/expedition/test_schedule.py deleted file mode 100644 index f4a8532e7..000000000 --- a/tests/expedition/test_schedule.py +++ /dev/null @@ -1,160 +0,0 @@ -from datetime import datetime, timedelta -from pathlib import Path - -import pyproj -import pytest - -from virtualship.errors import ScheduleError -from virtualship.expedition.do_expedition import _load_input_data -from virtualship.models import Location, Schedule, Waypoint -from virtualship.utils import _get_ship_config - -projection = pyproj.Geod(ellps="WGS84") - -expedition_dir = Path("expedition_dir") - - -def test_import_export_schedule(tmpdir) -> None: - out_path = tmpdir.join("schedule.yaml") - - # arbitrary time for testing - base_time = datetime.strptime("1950-01-01", "%Y-%m-%d") - - schedule = Schedule( - waypoints=[ - Waypoint(location=Location(0, 0), time=base_time, instrument=None), - Waypoint( - location=Location(1, 1), - time=base_time + timedelta(hours=1), - instrument=None, - ), - ] - ) - schedule.to_yaml(out_path) - - schedule2 = Schedule.from_yaml(out_path) - assert schedule == schedule2 - - -def test_verify_schedule() -> None: - schedule = Schedule( - waypoints=[ - Waypoint(location=Location(0, 0), time=datetime(2022, 1, 1, 1, 0, 0)), - Waypoint(location=Location(1, 0), time=datetime(2022, 1, 2, 1, 0, 0)), - ] - ) - - ship_config = _get_ship_config(expedition_dir) - - schedule.verify(ship_config.ship_speed_knots, None) - - -def test_get_instruments() -> None: - schedule = Schedule( - waypoints=[ - Waypoint(location=Location(0, 0), instrument=["CTD"]), - Waypoint(location=Location(1, 0), instrument=["XBT", "ARGO_FLOAT"]), - Waypoint(location=Location(1, 0), instrument=["CTD"]), - ] - ) - - assert set(instrument.name for instrument in schedule.get_instruments()) == { - "CTD", - "XBT", - "ARGO_FLOAT", - } - - -@pytest.mark.parametrize( - "schedule,check_space_time_region,error,match", - [ - pytest.param( - Schedule(waypoints=[]), - False, - ScheduleError, - "At least one waypoint must be provided.", - id="NoWaypoints", - ), - pytest.param( - Schedule( - waypoints=[ - Waypoint(location=Location(0, 0)), - Waypoint( - location=Location(1, 0), time=datetime(2022, 1, 1, 1, 0, 0) - ), - ] - ), - False, - ScheduleError, - "First waypoint must have a specified time.", - id="FirstWaypointHasTime", - ), - pytest.param( - Schedule( - waypoints=[ - Waypoint( - location=Location(0, 0), time=datetime(2022, 1, 2, 1, 0, 0) - ), - Waypoint(location=Location(0, 0)), - Waypoint( - location=Location(1, 0), time=datetime(2022, 1, 1, 1, 0, 0) - ), - ] - ), - False, - ScheduleError, - "Waypoint\\(s\\) : each waypoint should be timed after all previous waypoints", - id="SequentialWaypoints", - ), - pytest.param( - Schedule( - waypoints=[ - Waypoint( - location=Location(0, 0), time=datetime(2022, 1, 1, 1, 0, 0) - ), - Waypoint( - location=Location(1, 0), time=datetime(2022, 1, 1, 1, 1, 0) - ), - ] - ), - False, - ScheduleError, - "Waypoint planning is not valid: would arrive too late at waypoint number 2...", - id="NotEnoughTime", - ), - pytest.param( - Schedule( - waypoints=[ - Waypoint( - location=Location(0, 0), time=datetime(2022, 1, 1, 1, 0, 0) - ), - Waypoint( - location=Location(1, 0), time=datetime(2022, 1, 2, 1, 1, 0) - ), - ] - ), - True, - ScheduleError, - "space_time_region not found in schedule, please define it to fetch the data.", - id="NoSpaceTimeRegion", - ), - ], -) -def test_verify_schedule_errors( - schedule: Schedule, check_space_time_region: bool, error, match -) -> None: - ship_config = _get_ship_config(expedition_dir) - - input_data = _load_input_data( - expedition_dir, - schedule, - ship_config, - input_data=Path("expedition_dir/input_data"), - ) - - with pytest.raises(error, match=match): - schedule.verify( - ship_config.ship_speed_knots, - input_data, - check_space_time_region=check_space_time_region, - ) diff --git a/tests/expedition/test_ship_config.py b/tests/expedition/test_ship_config.py deleted file mode 100644 index 6444e9850..000000000 --- a/tests/expedition/test_ship_config.py +++ /dev/null @@ -1,126 +0,0 @@ -from pathlib import Path - -import pytest - -from virtualship.errors import ConfigError -from virtualship.models import Schedule, ShipConfig -from virtualship.utils import get_example_config, get_example_schedule - -expedition_dir = Path("expedition_dir") - - -@pytest.fixture -def schedule(tmp_file): - with open(tmp_file, "w") as file: - file.write(get_example_schedule()) - return Schedule.from_yaml(tmp_file) - - -@pytest.fixture -def schedule_no_xbt(schedule): - for waypoint in schedule.waypoints: - if waypoint.instrument and any( - instrument.name == "XBT" for instrument in waypoint.instrument - ): - waypoint.instrument = [ - instrument - for instrument in waypoint.instrument - if instrument.name != "XBT" - ] - - return schedule - - -@pytest.fixture -def ship_config(tmp_file): - with open(tmp_file, "w") as file: - file.write(get_example_config()) - return ShipConfig.from_yaml(tmp_file) - - -@pytest.fixture -def ship_config_no_xbt(ship_config): - delattr(ship_config, "xbt_config") - return ship_config - - -@pytest.fixture -def ship_config_no_ctd(ship_config): - delattr(ship_config, "ctd_config") - return ship_config - - -@pytest.fixture -def ship_config_no_ctd_bgc(ship_config): - delattr(ship_config, "ctd_bgc_config") - return ship_config - - -@pytest.fixture -def ship_config_no_argo_float(ship_config): - delattr(ship_config, "argo_float_config") - return ship_config - - -@pytest.fixture -def ship_config_no_drifter(ship_config): - delattr(ship_config, "drifter_config") - return ship_config - - -def test_import_export_ship_config(ship_config, tmp_file) -> None: - ship_config.to_yaml(tmp_file) - ship_config_2 = ShipConfig.from_yaml(tmp_file) - assert ship_config == ship_config_2 - - -def test_verify_ship_config(ship_config, schedule) -> None: - ship_config.verify(schedule) - - -def test_verify_ship_config_no_instrument(ship_config, schedule_no_xbt) -> None: - ship_config.verify(schedule_no_xbt) - - -@pytest.mark.parametrize( - "ship_config_fixture,error,match", - [ - pytest.param( - "ship_config_no_xbt", - ConfigError, - "Planning has a waypoint with XBT instrument, but configuration does not configure XBT.", - id="ShipConfigNoXBT", - ), - pytest.param( - "ship_config_no_ctd", - ConfigError, - "Planning has a waypoint with CTD instrument, but configuration does not configure CTD.", - id="ShipConfigNoCTD", - ), - pytest.param( - "ship_config_no_ctd_bgc", - ConfigError, - "Planning has a waypoint with CTD_BGC instrument, but configuration does not configure CTD_BGCs.", - id="ShipConfigNoCTD_BGC", - ), - pytest.param( - "ship_config_no_argo_float", - ConfigError, - "Planning has a waypoint with Argo float instrument, but configuration does not configure Argo floats.", - id="ShipConfigNoARGO_FLOAT", - ), - pytest.param( - "ship_config_no_drifter", - ConfigError, - "Planning has a waypoint with drifter instrument, but configuration does not configure drifters.", - id="ShipConfigNoDRIFTER", - ), - ], -) -def test_verify_ship_config_errors( - request, schedule, ship_config_fixture, error, match -) -> None: - ship_config = request.getfixturevalue(ship_config_fixture) - - with pytest.raises(error, match=match): - ship_config.verify(schedule) diff --git a/tests/expedition/test_simulate_schedule.py b/tests/expedition/test_simulate_schedule.py index 9eecd73dd..bad8c9ad3 100644 --- a/tests/expedition/test_simulate_schedule.py +++ b/tests/expedition/test_simulate_schedule.py @@ -7,7 +7,7 @@ ScheduleProblem, simulate_schedule, ) -from virtualship.models import Location, Schedule, ShipConfig, Waypoint +from virtualship.models import Expedition, Location, Schedule, Waypoint def test_simulate_schedule_feasible() -> None: @@ -15,16 +15,16 @@ def test_simulate_schedule_feasible() -> None: base_time = datetime.strptime("2022-01-01T00:00:00", "%Y-%m-%dT%H:%M:%S") projection = pyproj.Geod(ellps="WGS84") - ship_config = ShipConfig.from_yaml("expedition_dir/ship_config.yaml") - ship_config.ship_speed_knots = 10.0 - schedule = Schedule( + expedition = Expedition.from_yaml("expedition_dir/expedition.yaml") + expedition.ship_config.ship_speed_knots = 10.0 + expedition.schedule = Schedule( waypoints=[ Waypoint(location=Location(0, 0), time=base_time), Waypoint(location=Location(0.01, 0), time=base_time + timedelta(days=1)), ] ) - result = simulate_schedule(projection, ship_config, schedule) + result = simulate_schedule(projection, expedition) assert isinstance(result, ScheduleOk) @@ -34,23 +34,28 @@ def test_simulate_schedule_too_far() -> None: base_time = datetime.strptime("2022-01-01T00:00:00", "%Y-%m-%dT%H:%M:%S") projection = pyproj.Geod(ellps="WGS84") - ship_config = ShipConfig.from_yaml("expedition_dir/ship_config.yaml") - schedule = Schedule( + expedition = Expedition.from_yaml("expedition_dir/expedition.yaml") + expedition.ship_config.ship_speed_knots = 10.0 + expedition.schedule = Schedule( waypoints=[ Waypoint(location=Location(0, 0), time=base_time), Waypoint(location=Location(1.0, 0), time=base_time + timedelta(minutes=1)), ] ) - result = simulate_schedule(projection, ship_config, schedule) + result = simulate_schedule(projection, expedition) assert isinstance(result, ScheduleProblem) def test_time_in_minutes_in_ship_schedule() -> None: """Test whether the pydantic serializer picks up the time *in minutes* in the ship schedule.""" - ship_config = ShipConfig.from_yaml("expedition_dir/ship_config.yaml") - assert ship_config.adcp_config.period == timedelta(minutes=5) - assert ship_config.ctd_config.stationkeeping_time == timedelta(minutes=20) - assert ship_config.ctd_bgc_config.stationkeeping_time == timedelta(minutes=20) - assert ship_config.ship_underwater_st_config.period == timedelta(minutes=5) + instruments_config = Expedition.from_yaml( + "expedition_dir/expedition.yaml" + ).instruments_config + assert instruments_config.adcp_config.period == timedelta(minutes=5) + assert instruments_config.ctd_config.stationkeeping_time == timedelta(minutes=20) + assert instruments_config.ctd_bgc_config.stationkeeping_time == timedelta( + minutes=20 + ) + assert instruments_config.ship_underwater_st_config.period == timedelta(minutes=5) diff --git a/tests/test_mfp_to_yaml.py b/tests/test_mfp_to_yaml.py index d242d30a1..4eab16c29 100644 --- a/tests/test_mfp_to_yaml.py +++ b/tests/test_mfp_to_yaml.py @@ -3,7 +3,7 @@ import pandas as pd import pytest -from virtualship.models import Schedule +from virtualship.models import Expedition from virtualship.utils import mfp_to_yaml @@ -88,7 +88,7 @@ def test_mfp_to_yaml_success(request, fixture_name, tmp_path): """Test that mfp_to_yaml correctly processes a valid MFP file.""" valid_mfp_file = request.getfixturevalue(fixture_name) - yaml_output_path = tmp_path / "schedule.yaml" + yaml_output_path = tmp_path / "expedition.yaml" # Run function (No need to mock open() for YAML, real file is created) mfp_to_yaml(valid_mfp_file, yaml_output_path) @@ -97,9 +97,9 @@ def test_mfp_to_yaml_success(request, fixture_name, tmp_path): assert yaml_output_path.exists() # Load YAML and validate contents - data = Schedule.from_yaml(yaml_output_path) + data = Expedition.from_yaml(yaml_output_path) - assert len(data.waypoints) == 3 + assert len(data.schedule.waypoints) == 3 @pytest.mark.parametrize( @@ -138,7 +138,7 @@ def test_mfp_to_yaml_exceptions(request, fixture_name, error, match, tmp_path): """Test that mfp_to_yaml raises an error when input file is not valid.""" fixture = request.getfixturevalue(fixture_name) - yaml_output_path = tmp_path / "schedule.yaml" + yaml_output_path = tmp_path / "expedition.yaml" with pytest.raises(error, match=match): mfp_to_yaml(fixture, yaml_output_path) @@ -146,7 +146,7 @@ def test_mfp_to_yaml_exceptions(request, fixture_name, error, match, tmp_path): def test_mfp_to_yaml_extra_headers(unexpected_header_mfp_file, tmp_path): """Test that mfp_to_yaml prints a warning when extra columns are found.""" - yaml_output_path = tmp_path / "schedule.yaml" + yaml_output_path = tmp_path / "expedition.yaml" with pytest.warns(UserWarning, match="Found additional unexpected columns.*"): mfp_to_yaml(unexpected_header_mfp_file, yaml_output_path) diff --git a/tests/test_utils.py b/tests/test_utils.py index 4c6db8fc1..0dcebd794 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,26 +1,14 @@ -from virtualship.models import Schedule, ShipConfig -from virtualship.utils import get_example_config, get_example_schedule +from virtualship.models import Expedition +from virtualship.utils import get_example_expedition -def test_get_example_config(): - assert len(get_example_config()) > 0 +def test_get_example_expedition(): + assert len(get_example_expedition()) > 0 -def test_get_example_schedule(): - assert len(get_example_schedule()) > 0 - - -def test_valid_example_config(tmp_path): - path = tmp_path / "test.yaml" - with open(path, "w") as file: - file.write(get_example_config()) - - ShipConfig.from_yaml(path) - - -def test_valid_example_schedule(tmp_path): +def test_valid_example_expedition(tmp_path): path = tmp_path / "test.yaml" with open(path, "w") as file: - file.write(get_example_schedule()) + file.write(get_example_expedition()) - Schedule.from_yaml(path) + Expedition.from_yaml(path) From 2240b899913776a9c1ff8264ce233fc8870905a5 Mon Sep 17 00:00:00 2001 From: Nick Hodgskin <36369090+VeckoTheGecko@users.noreply.github.com> Date: Wed, 8 Oct 2025 14:43:45 +0200 Subject: [PATCH 02/97] Update link to website (#215) --- docs/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.md b/docs/index.md index 0210f2f50..d1cd43fc4 100644 --- a/docs/index.md +++ b/docs/index.md @@ -8,7 +8,7 @@ Home user-guide/index api/index contributing/index -VirtualShip Website +VirtualShip Website ``` ```{include} ../README.md From 63cbda5f756384311c32cbd03a2ec77d975cebc8 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 18 Aug 2025 21:01:36 +0100 Subject: [PATCH 03/97] first refactoring step, parent instrument classes --- src/virtualship/cli/_fetch.py | 11 ++ src/virtualship/instruments/adcp.py | 78 -------- src/virtualship/instruments/argo_float.py | 186 ------------------ src/virtualship/instruments/ctd.py | 164 ++++----------- src/virtualship/instruments/drifter.py | 113 ----------- .../instruments/ship_underwater_st.py | 76 ------- src/virtualship/instruments/xbt.py | 141 ------------- src/virtualship/models/instruments.py | 82 ++++++++ 8 files changed, 135 insertions(+), 716 deletions(-) delete mode 100644 src/virtualship/instruments/adcp.py delete mode 100644 src/virtualship/instruments/argo_float.py delete mode 100644 src/virtualship/instruments/drifter.py delete mode 100644 src/virtualship/instruments/ship_underwater_st.py delete mode 100644 src/virtualship/instruments/xbt.py create mode 100644 src/virtualship/models/instruments.py diff --git a/src/virtualship/cli/_fetch.py b/src/virtualship/cli/_fetch.py index 600083040..bf35fe2f0 100644 --- a/src/virtualship/cli/_fetch.py +++ b/src/virtualship/cli/_fetch.py @@ -86,6 +86,17 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None ) shutil.copyfile(path / EXPEDITION, download_folder / EXPEDITION) + #! + #### TODO + # ++ new logic here where iterates (?) through available instruments and determines whether download is required: + # ++ by conditions of: + # 1) whether it's in the schedule (and from this be able to call the right classes from the instruments directory?) and + #! 2) is there a clever way of not unnecessarily duplicating data downloads if instruments use the same?! + # (try with a version first where does them all in tow and then try and optimise...?) + + #! + ## TODO: move to generic bathymetry download which is done for all expeditions + if ( ( {"XBT", "CTD", "CDT_BGC", "SHIP_UNDERWATER_ST"} diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py deleted file mode 100644 index af2c285e5..000000000 --- a/src/virtualship/instruments/adcp.py +++ /dev/null @@ -1,78 +0,0 @@ -"""ADCP instrument.""" - -from pathlib import Path - -import numpy as np -from parcels import FieldSet, ParticleSet, ScipyParticle, Variable - -from virtualship.models import Spacetime - -# we specifically use ScipyParticle because we have many small calls to execute -# there is some overhead with JITParticle and this ends up being significantly faster -_ADCPParticle = ScipyParticle.add_variables( - [ - Variable("U", dtype=np.float32, initial=np.nan), - Variable("V", dtype=np.float32, initial=np.nan), - ] -) - - -def _sample_velocity(particle, fieldset, time): - particle.U, particle.V = fieldset.UV.eval( - time, particle.depth, particle.lat, particle.lon, applyConversion=False - ) - - -def simulate_adcp( - fieldset: FieldSet, - out_path: str | Path, - max_depth: float, - min_depth: float, - num_bins: int, - sample_points: list[Spacetime], -) -> None: - """ - Use Parcels to simulate an ADCP in a fieldset. - - :param fieldset: The fieldset to simulate the ADCP in. - :param out_path: The path to write the results to. - :param max_depth: Maximum depth the ADCP can measure. - :param min_depth: Minimum depth the ADCP can measure. - :param num_bins: How many samples to take in the complete range between max_depth and min_depth. - :param sample_points: The places and times to sample at. - """ - sample_points.sort(key=lambda p: p.time) - - bins = np.linspace(max_depth, min_depth, num_bins) - num_particles = len(bins) - particleset = ParticleSet.from_list( - fieldset=fieldset, - pclass=_ADCPParticle, - lon=np.full( - num_particles, 0.0 - ), # initial lat/lon are irrelevant and will be overruled later. - lat=np.full(num_particles, 0.0), - depth=bins, - time=0, # same for time - ) - - # define output file for the simulation - # outputdt set to infinite as we just want to write at the end of every call to 'execute' - out_file = particleset.ParticleFile(name=out_path, outputdt=np.inf) - - for point in sample_points: - particleset.lon_nextloop[:] = point.location.lon - particleset.lat_nextloop[:] = point.location.lat - particleset.time_nextloop[:] = fieldset.time_origin.reltime( - np.datetime64(point.time) - ) - - # perform one step using the particleset - # dt and runtime are set so exactly one step is made. - particleset.execute( - [_sample_velocity], - dt=1, - runtime=1, - verbose_progress=False, - output_file=out_file, - ) diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py deleted file mode 100644 index d09763673..000000000 --- a/src/virtualship/instruments/argo_float.py +++ /dev/null @@ -1,186 +0,0 @@ -"""Argo float instrument.""" - -import math -from dataclasses import dataclass -from datetime import datetime, timedelta -from pathlib import Path - -import numpy as np -from parcels import ( - AdvectionRK4, - FieldSet, - JITParticle, - ParticleSet, - StatusCode, - Variable, -) - -from virtualship.models import Spacetime - - -@dataclass -class ArgoFloat: - """Configuration for a single Argo float.""" - - spacetime: Spacetime - min_depth: float - max_depth: float - drift_depth: float - vertical_speed: float - cycle_days: float - drift_days: float - - -_ArgoParticle = JITParticle.add_variables( - [ - Variable("cycle_phase", dtype=np.int32, initial=0.0), - Variable("cycle_age", dtype=np.float32, initial=0.0), - Variable("drift_age", dtype=np.float32, initial=0.0), - Variable("salinity", dtype=np.float32, initial=np.nan), - Variable("temperature", dtype=np.float32, initial=np.nan), - Variable("min_depth", dtype=np.float32), - Variable("max_depth", dtype=np.float32), - Variable("drift_depth", dtype=np.float32), - Variable("vertical_speed", dtype=np.float32), - Variable("cycle_days", dtype=np.int32), - Variable("drift_days", dtype=np.int32), - ] -) - - -def _argo_float_vertical_movement(particle, fieldset, time): - if particle.cycle_phase == 0: - # Phase 0: Sinking with vertical_speed until depth is drift_depth - particle_ddepth += ( # noqa Parcels defines particle_* variables, which code checkers cannot know. - particle.vertical_speed * particle.dt - ) - if particle.depth + particle_ddepth <= particle.drift_depth: - particle_ddepth = particle.drift_depth - particle.depth - particle.cycle_phase = 1 - - elif particle.cycle_phase == 1: - # Phase 1: Drifting at depth for drifttime seconds - particle.drift_age += particle.dt - if particle.drift_age >= particle.drift_days * 86400: - particle.drift_age = 0 # reset drift_age for next cycle - particle.cycle_phase = 2 - - elif particle.cycle_phase == 2: - # Phase 2: Sinking further to max_depth - particle_ddepth += particle.vertical_speed * particle.dt - if particle.depth + particle_ddepth <= particle.max_depth: - particle_ddepth = particle.max_depth - particle.depth - particle.cycle_phase = 3 - - elif particle.cycle_phase == 3: - # Phase 3: Rising with vertical_speed until at surface - particle_ddepth -= particle.vertical_speed * particle.dt - particle.cycle_age += ( - particle.dt - ) # solve issue of not updating cycle_age during ascent - if particle.depth + particle_ddepth >= particle.min_depth: - particle_ddepth = particle.min_depth - particle.depth - particle.temperature = ( - math.nan - ) # reset temperature to NaN at end of sampling cycle - particle.salinity = math.nan # idem - particle.cycle_phase = 4 - else: - particle.temperature = fieldset.T[ - time, particle.depth, particle.lat, particle.lon - ] - particle.salinity = fieldset.S[ - time, particle.depth, particle.lat, particle.lon - ] - - elif particle.cycle_phase == 4: - # Phase 4: Transmitting at surface until cycletime is reached - if particle.cycle_age > particle.cycle_days * 86400: - particle.cycle_phase = 0 - particle.cycle_age = 0 - - if particle.state == StatusCode.Evaluate: - particle.cycle_age += particle.dt # update cycle_age - - -def _keep_at_surface(particle, fieldset, time): - # Prevent error when float reaches surface - if particle.state == StatusCode.ErrorThroughSurface: - particle.depth = particle.min_depth - particle.state = StatusCode.Success - - -def _check_error(particle, fieldset, time): - if particle.state >= 50: # This captures all Errors - particle.delete() - - -def simulate_argo_floats( - fieldset: FieldSet, - out_path: str | Path, - argo_floats: list[ArgoFloat], - outputdt: timedelta, - endtime: datetime | None, -) -> None: - """ - Use Parcels to simulate a set of Argo floats in a fieldset. - - :param fieldset: The fieldset to simulate the Argo floats in. - :param out_path: The path to write the results to. - :param argo_floats: A list of Argo floats to simulate. - :param outputdt: Interval which dictates the update frequency of file output during simulation - :param endtime: Stop at this time, or if None, continue until the end of the fieldset. - """ - DT = 10.0 # dt of Argo float simulation integrator - - if len(argo_floats) == 0: - print( - "No Argo floats provided. Parcels currently crashes when providing an empty particle set, so no argo floats simulation will be done and no files will be created." - ) - # TODO when Parcels supports it this check can be removed. - return - - # define parcel particles - argo_float_particleset = ParticleSet( - fieldset=fieldset, - pclass=_ArgoParticle, - lat=[argo.spacetime.location.lat for argo in argo_floats], - lon=[argo.spacetime.location.lon for argo in argo_floats], - depth=[argo.min_depth for argo in argo_floats], - time=[argo.spacetime.time for argo in argo_floats], - min_depth=[argo.min_depth for argo in argo_floats], - max_depth=[argo.max_depth for argo in argo_floats], - drift_depth=[argo.drift_depth for argo in argo_floats], - vertical_speed=[argo.vertical_speed for argo in argo_floats], - cycle_days=[argo.cycle_days for argo in argo_floats], - drift_days=[argo.drift_days for argo in argo_floats], - ) - - # define output file for the simulation - out_file = argo_float_particleset.ParticleFile( - name=out_path, outputdt=outputdt, chunks=[len(argo_float_particleset), 100] - ) - - # get earliest between fieldset end time and provide end time - fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) - if endtime is None: - actual_endtime = fieldset_endtime - elif endtime > fieldset_endtime: - print("WARN: Requested end time later than fieldset end time.") - actual_endtime = fieldset_endtime - else: - actual_endtime = np.timedelta64(endtime) - - # execute simulation - argo_float_particleset.execute( - [ - _argo_float_vertical_movement, - AdvectionRK4, - _keep_at_surface, - _check_error, - ], - endtime=actual_endtime, - dt=DT, - output_file=out_file, - verbose_progress=True, - ) diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 411850076..0d7294fb2 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -1,137 +1,57 @@ -"""CTD instrument.""" - from dataclasses import dataclass -from datetime import timedelta from pathlib import Path -import numpy as np -from parcels import FieldSet, JITParticle, ParticleSet, Variable +from virtualship.models import Spacetime, instruments -from virtualship.models import Spacetime +MYINSTRUMENT = "CTD" @dataclass class CTD: - """Configuration for a single CTD.""" + """CTD configuration.""" spacetime: Spacetime min_depth: float max_depth: float -_CTDParticle = JITParticle.add_variables( - [ - Variable("salinity", dtype=np.float32, initial=np.nan), - Variable("temperature", dtype=np.float32, initial=np.nan), - Variable("raising", dtype=np.int8, initial=0.0), # bool. 0 is False, 1 is True. - Variable("max_depth", dtype=np.float32), - Variable("min_depth", dtype=np.float32), - Variable("winch_speed", dtype=np.float32), - ] -) - - -def _sample_temperature(particle, fieldset, time): - particle.temperature = fieldset.T[time, particle.depth, particle.lat, particle.lon] - - -def _sample_salinity(particle, fieldset, time): - particle.salinity = fieldset.S[time, particle.depth, particle.lat, particle.lon] - - -def _ctd_cast(particle, fieldset, time): - # lowering - if particle.raising == 0: - particle_ddepth = -particle.winch_speed * particle.dt - if particle.depth + particle_ddepth < particle.max_depth: - particle.raising = 1 - particle_ddepth = -particle_ddepth - # raising - else: - particle_ddepth = particle.winch_speed * particle.dt - if particle.depth + particle_ddepth > particle.min_depth: - particle.delete() - - -def simulate_ctd( - fieldset: FieldSet, - out_path: str | Path, - ctds: list[CTD], - outputdt: timedelta, -) -> None: - """ - Use Parcels to simulate a set of CTDs in a fieldset. - - :param fieldset: The fieldset to simulate the CTDs in. - :param out_path: The path to write the results to. - :param ctds: A list of CTDs to simulate. - :param outputdt: Interval which dictates the update frequency of file output during simulation - :raises ValueError: Whenever provided CTDs, fieldset, are not compatible with this function. - """ - WINCH_SPEED = 1.0 # sink and rise speed in m/s - DT = 10.0 # dt of CTD simulation integrator - - if len(ctds) == 0: - print( - "No CTDs provided. Parcels currently crashes when providing an empty particle set, so no CTD simulation will be done and no files will be created." - ) - # TODO when Parcels supports it this check can be removed. - return - - fieldset_starttime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[0]) - fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) - - # deploy time for all ctds should be later than fieldset start time - if not all( - [np.datetime64(ctd.spacetime.time) >= fieldset_starttime for ctd in ctds] +# --------------- +# TODO: KERNELS +# --------------- + + +class CTDInputDataset(instruments.InputDataset): + """Input dataset for CTD instrument.""" + + def __init__(self): + """Initialise with instrument's name.""" + super().__init__(MYINSTRUMENT) + + def download_data(self, name: str) -> None: + """Download CTD data.""" + ... + + def get_dataset_path(self, name: str) -> Path: + """Get path to CTD dataset.""" + ... + + +class CTDInstrument(instruments.Instrument): + """CTD instrument class.""" + + def __init__( + self, + config, + input_dataset: CTDInputDataset, + kernels, ): - raise ValueError("CTD deployed before fieldset starts.") - - # depth the ctd will go to. shallowest between ctd max depth and bathymetry. - max_depths = [ - max( - ctd.max_depth, - fieldset.bathymetry.eval( - z=0, y=ctd.spacetime.location.lat, x=ctd.spacetime.location.lon, time=0 - ), - ) - for ctd in ctds - ] - - # CTD depth can not be too shallow, because kernel would break. - # This shallow is not useful anyway, no need to support. - if not all([max_depth <= -DT * WINCH_SPEED for max_depth in max_depths]): - raise ValueError( - f"CTD max_depth or bathymetry shallower than maximum {-DT * WINCH_SPEED}" - ) - - # define parcel particles - ctd_particleset = ParticleSet( - fieldset=fieldset, - pclass=_CTDParticle, - lon=[ctd.spacetime.location.lon for ctd in ctds], - lat=[ctd.spacetime.location.lat for ctd in ctds], - depth=[ctd.min_depth for ctd in ctds], - time=[ctd.spacetime.time for ctd in ctds], - max_depth=max_depths, - min_depth=[ctd.min_depth for ctd in ctds], - winch_speed=[WINCH_SPEED for _ in ctds], - ) - - # define output file for the simulation - out_file = ctd_particleset.ParticleFile(name=out_path, outputdt=outputdt) - - # execute simulation - ctd_particleset.execute( - [_sample_salinity, _sample_temperature, _ctd_cast], - endtime=fieldset_endtime, - dt=DT, - verbose_progress=False, - output_file=out_file, - ) - - # there should be no particles left, as they delete themselves when they resurface - if len(ctd_particleset.particledata) != 0: - raise ValueError( - "Simulation ended before CTD resurfaced. This most likely means the field time dimension did not match the simulation time span." - ) + """Initialise with instrument's name.""" + super().__init__(MYINSTRUMENT, config, input_dataset, kernels) + + def load_fieldset(self): + """Load fieldset.""" + ... + + def simulate(self): + """Simulate measurements.""" + ... diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py deleted file mode 100644 index 5aef240f1..000000000 --- a/src/virtualship/instruments/drifter.py +++ /dev/null @@ -1,113 +0,0 @@ -"""Drifter instrument.""" - -from dataclasses import dataclass -from datetime import datetime, timedelta -from pathlib import Path - -import numpy as np -from parcels import AdvectionRK4, FieldSet, JITParticle, ParticleSet, Variable - -from virtualship.models import Spacetime - - -@dataclass -class Drifter: - """Configuration for a single Drifter.""" - - spacetime: Spacetime - depth: float # depth at which it floats and samples - lifetime: timedelta | None # if none, lifetime is infinite - - -_DrifterParticle = JITParticle.add_variables( - [ - Variable("temperature", dtype=np.float32, initial=np.nan), - Variable("has_lifetime", dtype=np.int8), # bool - Variable("age", dtype=np.float32, initial=0.0), - Variable("lifetime", dtype=np.float32), - ] -) - - -def _sample_temperature(particle, fieldset, time): - particle.temperature = fieldset.T[time, particle.depth, particle.lat, particle.lon] - - -def _check_lifetime(particle, fieldset, time): - if particle.has_lifetime == 1: - particle.age += particle.dt - if particle.age >= particle.lifetime: - particle.delete() - - -def simulate_drifters( - fieldset: FieldSet, - out_path: str | Path, - drifters: list[Drifter], - outputdt: timedelta, - dt: timedelta, - endtime: datetime | None = None, -) -> None: - """ - Use Parcels to simulate a set of drifters in a fieldset. - - :param fieldset: The fieldset to simulate the Drifters in. - :param out_path: The path to write the results to. - :param drifters: A list of drifters to simulate. - :param outputdt: Interval which dictates the update frequency of file output during simulation. - :param dt: Dt for integration. - :param endtime: Stop at this time, or if None, continue until the end of the fieldset or until all drifters ended. If this is earlier than the last drifter ended or later than the end of the fieldset, a warning will be printed. - """ - if len(drifters) == 0: - print( - "No drifters provided. Parcels currently crashes when providing an empty particle set, so no drifter simulation will be done and no files will be created." - ) - # TODO when Parcels supports it this check can be removed. - return - - # define parcel particles - drifter_particleset = ParticleSet( - fieldset=fieldset, - pclass=_DrifterParticle, - lat=[drifter.spacetime.location.lat for drifter in drifters], - lon=[drifter.spacetime.location.lon for drifter in drifters], - depth=[drifter.depth for drifter in drifters], - time=[drifter.spacetime.time for drifter in drifters], - has_lifetime=[1 if drifter.lifetime is not None else 0 for drifter in drifters], - lifetime=[ - 0 if drifter.lifetime is None else drifter.lifetime.total_seconds() - for drifter in drifters - ], - ) - - # define output file for the simulation - out_file = drifter_particleset.ParticleFile( - name=out_path, outputdt=outputdt, chunks=[len(drifter_particleset), 100] - ) - - # get earliest between fieldset end time and provide end time - fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) - if endtime is None: - actual_endtime = fieldset_endtime - elif endtime > fieldset_endtime: - print("WARN: Requested end time later than fieldset end time.") - actual_endtime = fieldset_endtime - else: - actual_endtime = np.timedelta64(endtime) - - # execute simulation - drifter_particleset.execute( - [AdvectionRK4, _sample_temperature, _check_lifetime], - endtime=actual_endtime, - dt=dt, - output_file=out_file, - verbose_progress=True, - ) - - # if there are more particles left than the number of drifters with an indefinite endtime, warn the user - if len(drifter_particleset.particledata) > len( - [d for d in drifters if d.lifetime is None] - ): - print( - "WARN: Some drifters had a life time beyond the end time of the fieldset or the requested end time." - ) diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py deleted file mode 100644 index 7b08ad4bd..000000000 --- a/src/virtualship/instruments/ship_underwater_st.py +++ /dev/null @@ -1,76 +0,0 @@ -"""Ship salinity and temperature.""" - -from pathlib import Path - -import numpy as np -from parcels import FieldSet, ParticleSet, ScipyParticle, Variable - -from virtualship.models import Spacetime - -# we specifically use ScipyParticle because we have many small calls to execute -# there is some overhead with JITParticle and this ends up being significantly faster -_ShipSTParticle = ScipyParticle.add_variables( - [ - Variable("S", dtype=np.float32, initial=np.nan), - Variable("T", dtype=np.float32, initial=np.nan), - ] -) - - -# define function sampling Salinity -def _sample_salinity(particle, fieldset, time): - particle.S = fieldset.S[time, particle.depth, particle.lat, particle.lon] - - -# define function sampling Temperature -def _sample_temperature(particle, fieldset, time): - particle.T = fieldset.T[time, particle.depth, particle.lat, particle.lon] - - -def simulate_ship_underwater_st( - fieldset: FieldSet, - out_path: str | Path, - depth: float, - sample_points: list[Spacetime], -) -> None: - """ - Use Parcels to simulate underway data, measuring salinity and temperature at the given depth along the ship track in a fieldset. - - :param fieldset: The fieldset to simulate the sampling in. - :param out_path: The path to write the results to. - :param depth: The depth at which to measure. 0 is water surface, negative is into the water. - :param sample_points: The places and times to sample at. - """ - sample_points.sort(key=lambda p: p.time) - - particleset = ParticleSet.from_list( - fieldset=fieldset, - pclass=_ShipSTParticle, - lon=0.0, # initial lat/lon are irrelevant and will be overruled later - lat=0.0, - depth=depth, - time=0, # same for time - ) - - # define output file for the simulation - # outputdt set to infinie as we want to just want to write at the end of every call to 'execute' - out_file = particleset.ParticleFile(name=out_path, outputdt=np.inf) - - # iterate over each point, manually set lat lon time, then - # execute the particle set for one step, performing one set of measurement - for point in sample_points: - particleset.lon_nextloop[:] = point.location.lon - particleset.lat_nextloop[:] = point.location.lat - particleset.time_nextloop[:] = fieldset.time_origin.reltime( - np.datetime64(point.time) - ) - - # perform one step using the particleset - # dt and runtime are set so exactly one step is made. - particleset.execute( - [_sample_salinity, _sample_temperature], - dt=1, - runtime=1, - verbose_progress=False, - output_file=out_file, - ) diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py deleted file mode 100644 index 6d75be8c7..000000000 --- a/src/virtualship/instruments/xbt.py +++ /dev/null @@ -1,141 +0,0 @@ -"""XBT instrument.""" - -from dataclasses import dataclass -from datetime import timedelta -from pathlib import Path - -import numpy as np -from parcels import FieldSet, JITParticle, ParticleSet, Variable - -from virtualship.models import Spacetime - - -@dataclass -class XBT: - """Configuration for a single XBT.""" - - spacetime: Spacetime - min_depth: float - max_depth: float - fall_speed: float - deceleration_coefficient: float - - -_XBTParticle = JITParticle.add_variables( - [ - Variable("temperature", dtype=np.float32, initial=np.nan), - Variable("max_depth", dtype=np.float32), - Variable("min_depth", dtype=np.float32), - Variable("fall_speed", dtype=np.float32), - Variable("deceleration_coefficient", dtype=np.float32), - ] -) - - -def _sample_temperature(particle, fieldset, time): - particle.temperature = fieldset.T[time, particle.depth, particle.lat, particle.lon] - - -def _xbt_cast(particle, fieldset, time): - particle_ddepth = -particle.fall_speed * particle.dt - - # update the fall speed from the quadractic fall-rate equation - # check https://doi.org/10.5194/os-7-231-2011 - particle.fall_speed = ( - particle.fall_speed - 2 * particle.deceleration_coefficient * particle.dt - ) - - # delete particle if depth is exactly max_depth - if particle.depth == particle.max_depth: - particle.delete() - - # set particle depth to max depth if it's too deep - if particle.depth + particle_ddepth < particle.max_depth: - particle_ddepth = particle.max_depth - particle.depth - - -def simulate_xbt( - fieldset: FieldSet, - out_path: str | Path, - xbts: list[XBT], - outputdt: timedelta, -) -> None: - """ - Use Parcels to simulate a set of XBTs in a fieldset. - - :param fieldset: The fieldset to simulate the XBTs in. - :param out_path: The path to write the results to. - :param xbts: A list of XBTs to simulate. - :param outputdt: Interval which dictates the update frequency of file output during simulation - :raises ValueError: Whenever provided XBTs, fieldset, are not compatible with this function. - """ - DT = 10.0 # dt of XBT simulation integrator - - if len(xbts) == 0: - print( - "No XBTs provided. Parcels currently crashes when providing an empty particle set, so no XBT simulation will be done and no files will be created." - ) - # TODO when Parcels supports it this check can be removed. - return - - fieldset_starttime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[0]) - fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) - - # deploy time for all xbts should be later than fieldset start time - if not all( - [np.datetime64(xbt.spacetime.time) >= fieldset_starttime for xbt in xbts] - ): - raise ValueError("XBT deployed before fieldset starts.") - - # depth the xbt will go to. shallowest between xbt max depth and bathymetry. - max_depths = [ - max( - xbt.max_depth, - fieldset.bathymetry.eval( - z=0, y=xbt.spacetime.location.lat, x=xbt.spacetime.location.lon, time=0 - ), - ) - for xbt in xbts - ] - - # initial fall speeds - initial_fall_speeds = [xbt.fall_speed for xbt in xbts] - - # XBT depth can not be too shallow, because kernel would break. - # This shallow is not useful anyway, no need to support. - for max_depth, fall_speed in zip(max_depths, initial_fall_speeds, strict=False): - if not max_depth <= -DT * fall_speed: - raise ValueError( - f"XBT max_depth or bathymetry shallower than maximum {-DT * fall_speed}" - ) - - # define xbt particles - xbt_particleset = ParticleSet( - fieldset=fieldset, - pclass=_XBTParticle, - lon=[xbt.spacetime.location.lon for xbt in xbts], - lat=[xbt.spacetime.location.lat for xbt in xbts], - depth=[xbt.min_depth for xbt in xbts], - time=[xbt.spacetime.time for xbt in xbts], - max_depth=max_depths, - min_depth=[xbt.min_depth for xbt in xbts], - fall_speed=[xbt.fall_speed for xbt in xbts], - ) - - # define output file for the simulation - out_file = xbt_particleset.ParticleFile(name=out_path, outputdt=outputdt) - - # execute simulation - xbt_particleset.execute( - [_sample_temperature, _xbt_cast], - endtime=fieldset_endtime, - dt=DT, - verbose_progress=False, - output_file=out_file, - ) - - # there should be no particles left, as they delete themselves when they finish profiling - if len(xbt_particleset.particledata) != 0: - raise ValueError( - "Simulation ended before XBT finished profiling. This most likely means the field time dimension did not match the simulation time span." - ) diff --git a/src/virtualship/models/instruments.py b/src/virtualship/models/instruments.py new file mode 100644 index 000000000..8b5e701ef --- /dev/null +++ b/src/virtualship/models/instruments.py @@ -0,0 +1,82 @@ +import abc +from collections.abc import Callable +from pathlib import Path + +from yaspin import yaspin + +from virtualship.utils import ( + ship_spinner, +) + +# TODO +# how much detail needs to be fed into InputDataset (i.e. how much it differs per instrument) +# may impact whether need a child class (e.g. CTDInputDataset) as well as just InputDataset +# or whether it could just be fed a `name` ... ? + +# ++ abc.abstractmethods could be useful for testing purposes...e.g. will fail if an instrumnet implementation doesn't adhere to the `Instrument` class standards + + +class InputDataset(abc.ABC): + """Base class for instrument input datasets.""" + + def __init__(self, name): + """Initialise input dataset.""" + self.name = name + + @abc.abstractmethod + def download_data(self, name: str) -> None: + """Download data for the instrument.""" + pass + + @abc.abstractmethod + def get_dataset_path(self, name: str) -> Path: + """Get path to the dataset.""" + pass + + +class Instrument(abc.ABC): + """Base class for instruments.""" + + def __init__( + self, + name: str, + config, + input_dataset: InputDataset, + kernels: list[Callable], + ): + """Initialise instrument.""" + self.name = name + self.config = config + self.input_dataset = input_dataset + self.kernels = kernels + + @abc.abstractmethod + def load_fieldset(self): + """Load fieldset for simulation.""" + pass + + def get_output_path(self, output_dir: Path) -> Path: + """Get output path for results.""" + return output_dir / f"{self.name}.zarr" + + def run(self): + """Run instrument simulation.""" + with yaspin( + text=f"Simulating {self.name} measurements... ", + side="right", + spinner=ship_spinner, + ) as spinner: + self.simulate() + spinner.ok("✅") + + @abc.abstractmethod + def simulate(self): + """Simulate instrument measurements.""" + pass + + +# e.g. pseudo-code ... +# TODO: (necessary?) how to dynamically assemble list of all instruments defined so that new instruments can be added only by changes in one place...? +available_instruments: list = ... +# for instrument in available_instruments: +# MyInstrument(instrument) From b4fbf2be6090173c8127bd668a6482b41978d496 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 18 Aug 2025 21:07:41 +0100 Subject: [PATCH 04/97] ignore refactoring notes in gitignore --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 4efdfe453..bd70d1d56 100644 --- a/.gitignore +++ b/.gitignore @@ -178,3 +178,7 @@ src/virtualship/_version_setup.py .vscode/ .DS_Store + + +# Ignore temporary notes files for refactoring +_refactoring_notes/ From 2409517f0dd3302a3b8ef40b1514e6e497672cc4 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 18 Aug 2025 21:08:44 +0100 Subject: [PATCH 05/97] add note to remove upon completing v1 dev --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index bd70d1d56..d9903e718 100644 --- a/.gitignore +++ b/.gitignore @@ -181,4 +181,5 @@ src/virtualship/_version_setup.py # Ignore temporary notes files for refactoring +# TODO: remove when finished with v1 dev! _refactoring_notes/ From cfa7a0f840861ac96c65f31c3b7d41c13c994f79 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 17 Sep 2025 14:58:59 +0100 Subject: [PATCH 06/97] scratch inputdataset objects integration to _fetch --- .gitignore | 5 - src/virtualship/cli/_fetch.py | 184 ++++----------- src/virtualship/instruments/ctd.py | 59 +++-- src/virtualship/instruments/ctd_bgc.py | 80 +++++++ src/virtualship/instruments/master.py | 66 ++++++ src/virtualship/models/instruments.py | 87 +++++-- src/virtualship/models/schedule.py | 236 +++++++++++++++++++ src/virtualship/models/ship_config.py | 310 +++++++++++++++++++++++++ 8 files changed, 850 insertions(+), 177 deletions(-) create mode 100644 src/virtualship/instruments/master.py create mode 100644 src/virtualship/models/schedule.py create mode 100644 src/virtualship/models/ship_config.py diff --git a/.gitignore b/.gitignore index d9903e718..4efdfe453 100644 --- a/.gitignore +++ b/.gitignore @@ -178,8 +178,3 @@ src/virtualship/_version_setup.py .vscode/ .DS_Store - - -# Ignore temporary notes files for refactoring -# TODO: remove when finished with v1 dev! -_refactoring_notes/ diff --git a/src/virtualship/cli/_fetch.py b/src/virtualship/cli/_fetch.py index bf35fe2f0..127ee256a 100644 --- a/src/virtualship/cli/_fetch.py +++ b/src/virtualship/cli/_fetch.py @@ -2,10 +2,12 @@ import hashlib import shutil -from datetime import datetime, timedelta +from datetime import datetime from pathlib import Path from typing import TYPE_CHECKING +import copernicusmarine +from copernicusmarine.core_functions.credentials_utils import InvalidUsernameOrPassword from pydantic import BaseModel from virtualship.errors import IncompleteDownloadError @@ -19,11 +21,10 @@ from virtualship.models import SpaceTimeRegion import click -import copernicusmarine -from copernicusmarine.core_functions.credentials_utils import InvalidUsernameOrPassword import virtualship.cli._creds as creds from virtualship.utils import EXPEDITION +from virtualship.instruments.master import INSTRUMENTS DOWNLOAD_METADATA = "download_metadata.yaml" @@ -38,15 +39,13 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None be provided on prompt, via command line arguments, or via a YAML config file. Run `virtualship fetch` on an expedition for more info. """ - from virtualship.models import InstrumentType - if sum([username is None, password is None]) == 1: raise ValueError("Both username and password must be provided when using CLI.") path = Path(path) - data_folder = path / "data" - data_folder.mkdir(exist_ok=True) + data_dir = path / "data" + data_dir.mkdir(exist_ok=True) expedition = _get_expedition(path) @@ -61,7 +60,8 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None expedition.schedule.space_time_region ) - existing_download = get_existing_download(data_folder, space_time_region_hash) + # TODO: this (below) probably needs updating! + existing_download = get_existing_download(data_dir, space_time_region_hash) if existing_download is not None: click.echo( f"Data download for space-time region already completed ('{existing_download}')." @@ -69,7 +69,10 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None return creds_path = path / creds.CREDENTIALS_FILE - username, password = creds.get_credentials_flow(username, password, creds_path) + credentials = {} + credentials["username"], credentials["password"] = creds.get_credentials_flow( + username, password, creds_path + ) # Extract space_time_region details from the schedule spatial_range = expedition.schedule.space_time_region.spatial_range @@ -79,13 +82,46 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None instruments_in_schedule = expedition.schedule.get_instruments() # Create download folder and set download metadata - download_folder = data_folder / hash_to_filename(space_time_region_hash) + download_folder = data_dir / hash_to_filename(space_time_region_hash) download_folder.mkdir() DownloadMetadata(download_complete=False).to_yaml( download_folder / DOWNLOAD_METADATA ) shutil.copyfile(path / EXPEDITION, download_folder / EXPEDITION) + # bathymetry (required for all expeditions) + copernicusmarine.subset( + dataset_id="cmems_mod_glo_phy_my_0.083deg_static", + variables=["deptho"], + minimum_longitude=space_time_region.spatial_range.minimum_longitude, + maximum_longitude=space_time_region.spatial_range.maximum_longitude, + minimum_latitude=space_time_region.spatial_range.minimum_latitude, + maximum_latitude=space_time_region.spatial_range.maximum_latitude, + start_datetime=space_time_region.time_range.start_time, + end_datetime=space_time_region.time_range.start_time, + minimum_depth=abs(space_time_region.spatial_range.minimum_depth), + maximum_depth=abs(space_time_region.spatial_range.maximum_depth), + output_filename="bathymetry.nc", + output_directory=download_folder, + username=credentials["username"], + password=credentials["password"], + overwrite=True, + coordinates_selection_method="outside", + ) + + # keep only instruments in INTSTRUMENTS which are in schedule + filtered_instruments = { + k: v for k, v in INSTRUMENTS.items() if k in instruments_in_schedule + } + + # iterate across instruments and download data based on space_time_region + for _, instrument in filtered_instruments.items(): + try: + instrument["input_class"]( + data_dir=download_folder, + credentials=credentials, + space_time_region=space_time_region, + ) #! #### TODO # ++ new logic here where iterates (?) through available instruments and determines whether download is required: @@ -198,127 +234,7 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None shutil.rmtree(download_folder) raise e - click.echo("Drifter data download based on space-time region completed.") - - if InstrumentType.ARGO_FLOAT in instruments_in_schedule: - print("Argo float data will be downloaded. Please wait...") - argo_download_dict = { - "UVdata": { - "dataset_id": "cmems_mod_glo_phy-cur_anfc_0.083deg_PT6H-i", - "variables": ["uo", "vo"], - "output_filename": "argo_float_uv.nc", - }, - "Sdata": { - "dataset_id": "cmems_mod_glo_phy-so_anfc_0.083deg_PT6H-i", - "variables": ["so"], - "output_filename": "argo_float_s.nc", - }, - "Tdata": { - "dataset_id": "cmems_mod_glo_phy-thetao_anfc_0.083deg_PT6H-i", - "variables": ["thetao"], - "output_filename": "argo_float_t.nc", - }, - } - - # Iterate over all datasets and download each based on space_time_region - try: - for dataset in argo_download_dict.values(): - copernicusmarine.subset( - dataset_id=dataset["dataset_id"], - variables=dataset["variables"], - minimum_longitude=spatial_range.minimum_longitude - 3.0, - maximum_longitude=spatial_range.maximum_longitude + 3.0, - minimum_latitude=spatial_range.minimum_latitude - 3.0, - maximum_latitude=spatial_range.maximum_latitude + 3.0, - start_datetime=start_datetime, - end_datetime=end_datetime + timedelta(days=21), - minimum_depth=abs(1), - maximum_depth=abs(spatial_range.maximum_depth), - output_filename=dataset["output_filename"], - output_directory=download_folder, - username=username, - password=password, - overwrite=True, - coordinates_selection_method="outside", - ) - except InvalidUsernameOrPassword as e: - shutil.rmtree(download_folder) - raise e - - click.echo("Argo_float data download based on space-time region completed.") - - if InstrumentType.CTD_BGC in instruments_in_schedule: - print("CTD_BGC data will be downloaded. Please wait...") - - ctd_bgc_download_dict = { - "o2data": { - "dataset_id": "cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m", - "variables": ["o2"], - "output_filename": "ctd_bgc_o2.nc", - }, - "chlorodata": { - "dataset_id": "cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m", - "variables": ["chl"], - "output_filename": "ctd_bgc_chl.nc", - }, - "nitratedata": { - "dataset_id": "cmems_mod_glo_bgc-nut_anfc_0.25deg_P1D-m", - "variables": ["no3"], - "output_filename": "ctd_bgc_no3.nc", - }, - "phosphatedata": { - "dataset_id": "cmems_mod_glo_bgc-nut_anfc_0.25deg_P1D-m", - "variables": ["po4"], - "output_filename": "ctd_bgc_po4.nc", - }, - "phdata": { - "dataset_id": "cmems_mod_glo_bgc-car_anfc_0.25deg_P1D-m", - "variables": ["ph"], - "output_filename": "ctd_bgc_ph.nc", - }, - "phytoplanktondata": { - "dataset_id": "cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m", - "variables": ["phyc"], - "output_filename": "ctd_bgc_phyc.nc", - }, - "zooplanktondata": { - "dataset_id": "cmems_mod_glo_bgc-plankton_anfc_0.25deg_P1D-m", - "variables": ["zooc"], - "output_filename": "ctd_bgc_zooc.nc", - }, - "primaryproductiondata": { - "dataset_id": "cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m", - "variables": ["nppv"], - "output_filename": "ctd_bgc_nppv.nc", - }, - } - - # Iterate over all datasets and download each based on space_time_region - try: - for dataset in ctd_bgc_download_dict.values(): - copernicusmarine.subset( - dataset_id=dataset["dataset_id"], - variables=dataset["variables"], - minimum_longitude=spatial_range.minimum_longitude - 3.0, - maximum_longitude=spatial_range.maximum_longitude + 3.0, - minimum_latitude=spatial_range.minimum_latitude - 3.0, - maximum_latitude=spatial_range.maximum_latitude + 3.0, - start_datetime=start_datetime, - end_datetime=end_datetime + timedelta(days=21), - minimum_depth=abs(1), - maximum_depth=abs(spatial_range.maximum_depth), - output_filename=dataset["output_filename"], - output_directory=download_folder, - username=username, - password=password, - overwrite=True, - coordinates_selection_method="outside", - ) - except InvalidUsernameOrPassword as e: - shutil.rmtree(download_folder) - raise e - - click.echo("CTD_BGC data download based on space-time region completed.") + click.echo(f"{instrument.name} data download completed.") # TODO complete_download(download_folder) @@ -386,11 +302,9 @@ def from_yaml(cls, file_path: str | Path) -> DownloadMetadata: return _generic_load_yaml(file_path, cls) -def get_existing_download( - data_folder: Path, space_time_region_hash: str -) -> Path | None: +def get_existing_download(data_dir: Path, space_time_region_hash: str) -> Path | None: """Check if a download has already been completed. If so, return the path for existing download.""" - for download_path in data_folder.rglob("*"): + for download_path in data_dir.rglob("*"): try: hash = filename_to_hash(download_path.name) except ValueError: diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 0d7294fb2..624256cb7 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -1,15 +1,17 @@ from dataclasses import dataclass -from pathlib import Path +from typing import ClassVar from virtualship.models import Spacetime, instruments -MYINSTRUMENT = "CTD" +## TODO: __init__.py will also need updating! +# + therefore instructions for adding new instruments will also involve adding to __init__.py as well as the new instrument script + update InstrumentType in instruments.py @dataclass class CTD: """CTD configuration.""" + name: ClassVar[str] = "CTD" spacetime: Spacetime min_depth: float max_depth: float @@ -23,17 +25,36 @@ class CTD: class CTDInputDataset(instruments.InputDataset): """Input dataset for CTD instrument.""" - def __init__(self): - """Initialise with instrument's name.""" - super().__init__(MYINSTRUMENT) - - def download_data(self, name: str) -> None: - """Download CTD data.""" - ... + DOWNLOAD_BUFFERS: ClassVar[dict] = { + "latlon_degrees": 0.0, + "days": 0.0, + } # CTD data requires no buffers - def get_dataset_path(self, name: str) -> Path: - """Get path to CTD dataset.""" - ... + def __init__(self, data_dir, credentials, space_time_region): + """Initialise with instrument's name.""" + super().__init__( + CTD.name, + self.DOWNLOAD_BUFFERS["latlon_degrees"], + self.DOWNLOAD_BUFFERS["days"], + data_dir, + credentials, + space_time_region, + ) + + def get_datasets_dict(self) -> dict: + """Get variable specific args for instrument.""" + return { + "Sdata": { + "dataset_id": "cmems_mod_glo_phy-so_anfc_0.083deg_PT6H-i", + "variables": ["so"], + "output_filename": f"{self.name}_s.nc", + }, + "Tdata": { + "dataset_id": "cmems_mod_glo_phy-thetao_anfc_0.083deg_PT6H-i", + "variables": ["thetao"], + "output_filename": f"{self.name}_t.nc", + }, + } class CTDInstrument(instruments.Instrument): @@ -42,16 +63,18 @@ class CTDInstrument(instruments.Instrument): def __init__( self, config, - input_dataset: CTDInputDataset, + input_dataset, kernels, ): """Initialise with instrument's name.""" - super().__init__(MYINSTRUMENT, config, input_dataset, kernels) - - def load_fieldset(self): - """Load fieldset.""" - ... + super().__init__(CTD.name, config, input_dataset, kernels) def simulate(self): """Simulate measurements.""" ... + + +# # [PSEUDO-CODE] example implementation for reference +# ctd = CTDInstrument(config=CTD, data_dir=..., kernels=...) + +# ctd.simulate(...) diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index fde92ca10..6b9b2f298 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -1,3 +1,83 @@ +# from dataclasses import dataclass +# from typing import ClassVar + +# from virtualship.models import Spacetime, instruments + +# MYINSTRUMENT = "CTD_BGC" + + +# @dataclass +# class CTD_BGC: +# """CTD_BGC configuration.""" + +# spacetime: Spacetime +# min_depth: float +# max_depth: float + + +# # --------------- +# # TODO: KERNELS +# # --------------- + + +# class CTD_BGCInputDataset(instruments.InputDataset): +# """Input dataset object for CTD_BGC instrument.""" + +# DOWNLOAD_BUFFERS: ClassVar[dict] = { +# "latlon_degrees": 0.0, +# "days": 0.0, +# } # CTD_BGC data requires no buffers + +# def __init__(self, data_dir, credentials, space_time_region): +# """Initialise with instrument's name.""" +# super().__init__( +# MYINSTRUMENT, +# self.DOWNLOAD_BUFFERS["latlon_degrees"], +# self.DOWNLOAD_BUFFERS["days"], +# data_dir, +# credentials, +# space_time_region, +# ) + +# def datasets_dir(self) -> dict: +# """Variable specific args for instrument.""" +# return { +# "o2data": { +# "dataset_id": "cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m", +# "variables": ["o2"], +# "output_filename": "ctd_bgc_o2.nc", +# }, +# "chlorodata": { +# "dataset_id": "cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m", +# "variables": ["chl"], +# "output_filename": "ctd_bgc_chloro.nc", +# }, +# } + + +# class CTD_BGCInstrument(instruments.Instrument): +# """CTD_BGC instrument class.""" + +# def __init__( +# self, +# config, +# input_dataset, +# kernels, +# ): +# """Initialise with instrument's name.""" +# super().__init__(MYINSTRUMENT, config, input_dataset, kernels) + +# def simulate(self): +# """Simulate measurements.""" +# ... + + +# # # [PSEUDO-CODE] example implementation for reference +# # ctd = CTD_BGCInstrument(config=CTD_BGC, data_dir=..., kernels=...) + +# # ctd.simulate(...) + + """CTD_BGC instrument.""" from dataclasses import dataclass diff --git a/src/virtualship/instruments/master.py b/src/virtualship/instruments/master.py new file mode 100644 index 000000000..4705c8e3a --- /dev/null +++ b/src/virtualship/instruments/master.py @@ -0,0 +1,66 @@ +# + +# TODO: temporary measure so as not to have to overhaul the InstrumentType class logic in one go +#! And also to avoid breaking other parts of the codebase which rely on InstrumentType when for now just working on fetch +# TODO: ideally this can evaporate... +# TODO: discuss to see if there's a better option...! + +from enum import Enum + +# and so on ... +# from virtualship.instruments.ctd import CTDInputDataset, CTDInstrument + + +class InstrumentType(Enum): + """Types of the instruments.""" + + CTD = "CTD" + # CTD_BGC = "CTD_BGC" + # DRIFTER = "DRIFTER" + # ARGO_FLOAT = "ARGO_FLOAT" + # XBT = "XBT" + + # # TODO: should underway also be handled here?! + # ADCP = "ADCP" + # UNDERWAY_ST = "UNDERWAY_ST" + + +# replace with imports instead... +class CTDInputDataset: + """Input dataset class for CTD instrument.""" + + pass + + +class CTDInstrument: + """Instrument class for CTD instrument.""" + + pass + + +INSTRUMENTS = { + inst: { + "input_class": globals()[f"{inst.value}InputDataset"], + "instrument_class": globals()[f"{inst.value}Instrument"], + } + for inst in InstrumentType + if f"{inst.value}InputDataset" in globals() + and f"{inst.value}Instrument" in globals() +} + + +# INSTRUMENTS = { +# InstrumentType.CTD: { +# "input_class": CTDInputDataset, +# "instrument_class": CTDInstrument, +# } +# # and so on for other instruments... +# } + +# INSTRUMENTS = { +# "InstrumentType.CTD": { +# "input_class": "test", +# "instrument_class": "test", +# } +# # and so on for other instruments... +# } diff --git a/src/virtualship/models/instruments.py b/src/virtualship/models/instruments.py index 8b5e701ef..cddebd85f 100644 --- a/src/virtualship/models/instruments.py +++ b/src/virtualship/models/instruments.py @@ -1,37 +1,86 @@ import abc from collections.abc import Callable +from datetime import timedelta from pathlib import Path +import copernicusmarine from yaspin import yaspin -from virtualship.utils import ( - ship_spinner, -) +from virtualship.models.space_time_region import SpaceTimeRegion +from virtualship.utils import ship_spinner -# TODO +# TODO list START # how much detail needs to be fed into InputDataset (i.e. how much it differs per instrument) # may impact whether need a child class (e.g. CTDInputDataset) as well as just InputDataset # or whether it could just be fed a `name` ... ? # ++ abc.abstractmethods could be useful for testing purposes...e.g. will fail if an instrumnet implementation doesn't adhere to the `Instrument` class standards +# ++ discussion point with others, do we think it's okay to overhaul the data downloading so that each instrument has it's own files, rather than sharing data? +# ++ it's a cleaner way of making the whole repo more modular, i.e. have higher order logic for defining data downloads and housing all instrument logic in one place... +# ++ may even not matter so much considering working towards cloud integration... + we are not looking to optimise performance...? +# ++ OR, for now work on it in this way and then at the end make some clever changes to consolidate to minimum number of files dependent on instrument selections...? +# TODO list END + class InputDataset(abc.ABC): """Base class for instrument input datasets.""" - def __init__(self, name): + def __init__( + self, + name: str, + latlon_buffer: float, + datetime_buffer: float, + data_dir: str, + credentials: dict, + space_time_region: SpaceTimeRegion, + ): """Initialise input dataset.""" self.name = name + self.latlon_buffer = latlon_buffer + self.datetime_buffer = datetime_buffer + self.data_dir = data_dir + self.credentials = credentials + self.space_time_region = space_time_region @abc.abstractmethod - def download_data(self, name: str) -> None: - """Download data for the instrument.""" - pass - - @abc.abstractmethod - def get_dataset_path(self, name: str) -> Path: - """Get path to the dataset.""" - pass + def get_datasets_dict(self) -> dict: + """Get parameters for instrument's variable(s) specific data download.""" + ... + + def download_data(self) -> None: + """Download data for the instrument using copernicusmarine.""" + parameter_args = dict( + minimum_longitude=self.space_time_region.spatial_range.minimum_longitude + - self.latlon_buffer, + maximum_longitude=self.space_time_region.spatial_range.maximum_longitude + + self.latlon_buffer, + minimum_latitude=self.space_time_region.spatial_range.minimum_latitude + - self.latlon_buffer, + maximum_latitude=self.space_time_region.spatial_range.maximum_latitude + + self.latlon_buffer, + start_datetime=self.space_time_region.time_range.start_time, + end_datetime=self.space_time_region.time_range.end_time + + timedelta(days=self.datetime_buffer), + minimum_depth=abs(self.space_time_region.spatial_range.minimum_depth), + maximum_depth=abs(self.space_time_region.spatial_range.maximum_depth), + output_directory=self.data_dir, + username=self.credentials["username"], + password=self.credentials["password"], + overwrite=True, + coordinates_selection_method="outside", + ) + + datasets_args = self.get_datasets_dict() + + for dataset in datasets_args.values(): + download_args = {**parameter_args, **dataset} + copernicusmarine.subset(**download_args) + + # def get_fieldset_paths(self) -> list: + # """List of paths for instrument's (downloaded) input data.""" + + # ... class Instrument(abc.ABC): @@ -47,13 +96,13 @@ def __init__( """Initialise instrument.""" self.name = name self.config = config - self.input_dataset = input_dataset + self.input_data = input_dataset self.kernels = kernels - @abc.abstractmethod - def load_fieldset(self): - """Load fieldset for simulation.""" - pass + # def load_fieldset(self): + # """Load fieldset for simulation.""" + # # paths = self.input_data.get_fieldset_paths() + # ... def get_output_path(self, output_dir: Path) -> Path: """Get output path for results.""" @@ -72,7 +121,7 @@ def run(self): @abc.abstractmethod def simulate(self): """Simulate instrument measurements.""" - pass + ... # e.g. pseudo-code ... diff --git a/src/virtualship/models/schedule.py b/src/virtualship/models/schedule.py new file mode 100644 index 000000000..091c23b49 --- /dev/null +++ b/src/virtualship/models/schedule.py @@ -0,0 +1,236 @@ +"""Schedule class.""" + +from __future__ import annotations + +import itertools +from datetime import datetime, timedelta +from pathlib import Path +from typing import TYPE_CHECKING + +import pydantic +import pyproj +import yaml + +from virtualship.errors import ScheduleError + +from .instruments import InstrumentType +from .location import Location +from .space_time_region import SpaceTimeRegion + +if TYPE_CHECKING: + from parcels import FieldSet + + from virtualship.expedition.input_data import InputData + +projection: pyproj.Geod = pyproj.Geod(ellps="WGS84") + + +class Waypoint(pydantic.BaseModel): + """A Waypoint to sail to with an optional time and an optional instrument.""" + + location: Location + time: datetime | None = None + instrument: InstrumentType | list[InstrumentType] | None = None + + @pydantic.field_serializer("instrument") + def serialize_instrument(self, instrument): + """Ensure InstrumentType is serialized as a string (or list of strings).""" + if isinstance(instrument, list): + return [inst.value for inst in instrument] + return instrument.value if instrument else None + + +class Schedule(pydantic.BaseModel): + """Schedule of the virtual ship.""" + + waypoints: list[Waypoint] + space_time_region: SpaceTimeRegion | None = None + + model_config = pydantic.ConfigDict(extra="forbid") + + def to_yaml(self, file_path: str | Path) -> None: + """ + Write schedule to yaml file. + + :param file_path: Path to the file to write to. + """ + with open(file_path, "w") as file: + yaml.dump( + self.model_dump( + by_alias=True, + ), + file, + ) + + @classmethod + def from_yaml(cls, file_path: str | Path) -> Schedule: + """ + Load schedule from yaml file. + + :param file_path: Path to the file to load from. + :returns: The schedule. + """ + with open(file_path) as file: + data = yaml.safe_load(file) + return Schedule(**data) + + def get_instruments(self) -> set[InstrumentType]: + """ + Retrieve a set of unique instruments used in the schedule. + + This method iterates through all waypoints in the schedule and collects + the instruments associated with each waypoint. It returns a set of unique + instruments, either as objects or as names. + + :raises CheckpointError: If the past waypoints in the given schedule + have been changed compared to the checkpoint. + :return: set: A set of unique instruments used in the schedule. + + """ + instruments_in_schedule = [] + for waypoint in self.waypoints: + if waypoint.instrument: + for instrument in waypoint.instrument: + if instrument: + instruments_in_schedule.append(instrument) + return set(instruments_in_schedule) + + def verify( + self, + ship_speed: float, + input_data: InputData | None, + *, + check_space_time_region: bool = False, + ignore_missing_fieldsets: bool = False, + ) -> None: + """ + Verify the feasibility and correctness of the schedule's waypoints. + + This method checks various conditions to ensure the schedule is valid: + 1. At least one waypoint is provided. + 2. The first waypoint has a specified time. + 3. Waypoint times are in ascending order. + 4. All waypoints are in water (not on land). + 5. The ship can arrive on time at each waypoint given its speed. + + :param ship_speed: The ship's speed in knots. + :param input_data: An InputData object containing fieldsets used to check if waypoints are on water. + :param check_space_time_region: whether to check for missing space_time_region. + :param ignore_missing_fieldsets: whether to ignore warning for missing field sets. + :raises PlanningError: If any of the verification checks fail, indicating infeasible or incorrect waypoints. + :raises NotImplementedError: If an instrument in the schedule is not implemented. + :return: None. The method doesn't return a value but raises exceptions if verification fails. + """ + print("\nVerifying route... ") + + if check_space_time_region and self.space_time_region is None: + raise ScheduleError( + "space_time_region not found in schedule, please define it to fetch the data." + ) + + if len(self.waypoints) == 0: + raise ScheduleError("At least one waypoint must be provided.") + + # check first waypoint has a time + if self.waypoints[0].time is None: + raise ScheduleError("First waypoint must have a specified time.") + + # check waypoint times are in ascending order + timed_waypoints = [wp for wp in self.waypoints if wp.time is not None] + checks = [ + next.time >= cur.time for cur, next in itertools.pairwise(timed_waypoints) + ] + if not all(checks): + invalid_i = [i for i, c in enumerate(checks) if c] + raise ScheduleError( + f"Waypoint(s) {', '.join(f'#{i + 1}' for i in invalid_i)}: each waypoint should be timed after all previous waypoints", + ) + + # check if all waypoints are in water + # this is done by picking an arbitrary provided fieldset and checking if UV is not zero + + # get all available fieldsets + available_fieldsets = [] + if input_data is not None: + fieldsets = [ + input_data.adcp_fieldset, + input_data.argo_float_fieldset, + input_data.ctd_fieldset, + input_data.drifter_fieldset, + input_data.ship_underwater_st_fieldset, + ] + for fs in fieldsets: + if fs is not None: + available_fieldsets.append(fs) + + # check if there are any fieldsets, else it's an error + if len(available_fieldsets) == 0: + if not ignore_missing_fieldsets: + print( + "Cannot verify because no fieldsets have been loaded. This is probably " + "because you are not using any instruments in your schedule. This is not a problem, " + "but carefully check your waypoint locations manually." + ) + + else: + # pick any + fieldset = available_fieldsets[0] + # get waypoints with 0 UV + land_waypoints = [ + (wp_i, wp) + for wp_i, wp in enumerate(self.waypoints) + if _is_on_land_zero_uv(fieldset, wp) + ] + # raise an error if there are any + if len(land_waypoints) > 0: + raise ScheduleError( + f"The following waypoints are on land: {['#' + str(wp_i) + ' ' + str(wp) for (wp_i, wp) in land_waypoints]}" + ) + + # check that ship will arrive on time at each waypoint (in case no unexpected event happen) + time = self.waypoints[0].time + for wp_i, (wp, wp_next) in enumerate( + zip(self.waypoints, self.waypoints[1:], strict=False) + ): + if wp.instrument is InstrumentType.CTD: + time += timedelta(minutes=20) + + geodinv: tuple[float, float, float] = projection.inv( + wp.location.lon, + wp.location.lat, + wp_next.location.lon, + wp_next.location.lat, + ) + distance = geodinv[2] + + time_to_reach = timedelta(seconds=distance / ship_speed * 3600 / 1852) + arrival_time = time + time_to_reach + + if wp_next.time is None: + time = arrival_time + elif arrival_time > wp_next.time: + raise ScheduleError( + f"Waypoint planning is not valid: would arrive too late at waypoint number {wp_i + 2}. " + f"location: {wp_next.location} time: {wp_next.time} instrument: {wp_next.instrument}" + ) + else: + time = wp_next.time + + print("... All good to go!") + + +def _is_on_land_zero_uv(fieldset: FieldSet, waypoint: Waypoint) -> bool: + """ + Check if waypoint is on land by assuming zero velocity means land. + + :param fieldset: The fieldset to sample the velocity from. + :param waypoint: The waypoint to check. + :returns: If the waypoint is on land. + """ + return fieldset.UV.eval( + 0, + fieldset.gridset.grids[0].depth[0], + waypoint.location.lat, + waypoint.location.lon, + applyConversion=False, + ) == (0.0, 0.0) diff --git a/src/virtualship/models/ship_config.py b/src/virtualship/models/ship_config.py new file mode 100644 index 000000000..61d3d3905 --- /dev/null +++ b/src/virtualship/models/ship_config.py @@ -0,0 +1,310 @@ +"""ShipConfig and supporting classes.""" + +from __future__ import annotations + +from datetime import timedelta +from pathlib import Path +from typing import TYPE_CHECKING + +import pydantic +import yaml + +from virtualship.errors import ConfigError +from virtualship.models.instruments import InstrumentType +from virtualship.utils import _validate_numeric_mins_to_timedelta + +if TYPE_CHECKING: + from .schedule import Schedule + + +class ArgoFloatConfig(pydantic.BaseModel): + """Configuration for argos floats.""" + + min_depth_meter: float = pydantic.Field(le=0.0) + max_depth_meter: float = pydantic.Field(le=0.0) + drift_depth_meter: float = pydantic.Field(le=0.0) + vertical_speed_meter_per_second: float = pydantic.Field(lt=0.0) + cycle_days: float = pydantic.Field(gt=0.0) + drift_days: float = pydantic.Field(gt=0.0) + + +class ADCPConfig(pydantic.BaseModel): + """Configuration for ADCP instrument.""" + + max_depth_meter: float = pydantic.Field(le=0.0) + num_bins: int = pydantic.Field(gt=0.0) + period: timedelta = pydantic.Field( + serialization_alias="period_minutes", + validation_alias="period_minutes", + gt=timedelta(), + ) + + model_config = pydantic.ConfigDict(populate_by_name=True) + + @pydantic.field_serializer("period") + def _serialize_period(self, value: timedelta, _info): + return value.total_seconds() / 60.0 + + @pydantic.field_validator("period", mode="before") + def _validate_period(cls, value: int | float | timedelta) -> timedelta: + return _validate_numeric_mins_to_timedelta(value) + + +class CTDConfig(pydantic.BaseModel): + """Configuration for CTD instrument.""" + + stationkeeping_time: timedelta = pydantic.Field( + serialization_alias="stationkeeping_time_minutes", + validation_alias="stationkeeping_time_minutes", + gt=timedelta(), + ) + min_depth_meter: float = pydantic.Field(le=0.0) + max_depth_meter: float = pydantic.Field(le=0.0) + + model_config = pydantic.ConfigDict(populate_by_name=True) + + @pydantic.field_serializer("stationkeeping_time") + def _serialize_stationkeeping_time(self, value: timedelta, _info): + return value.total_seconds() / 60.0 + + @pydantic.field_validator("stationkeeping_time", mode="before") + def _validate_stationkeeping_time(cls, value: int | float | timedelta) -> timedelta: + return _validate_numeric_mins_to_timedelta(value) + + +class CTD_BGCConfig(pydantic.BaseModel): + """Configuration for CTD_BGC instrument.""" + + stationkeeping_time: timedelta = pydantic.Field( + serialization_alias="stationkeeping_time_minutes", + validation_alias="stationkeeping_time_minutes", + gt=timedelta(), + ) + min_depth_meter: float = pydantic.Field(le=0.0) + max_depth_meter: float = pydantic.Field(le=0.0) + + model_config = pydantic.ConfigDict(populate_by_name=True) + + @pydantic.field_serializer("stationkeeping_time") + def _serialize_stationkeeping_time(self, value: timedelta, _info): + return value.total_seconds() / 60.0 + + @pydantic.field_validator("stationkeeping_time", mode="before") + def _validate_stationkeeping_time(cls, value: int | float | timedelta) -> timedelta: + return _validate_numeric_mins_to_timedelta(value) + + +class ShipUnderwaterSTConfig(pydantic.BaseModel): + """Configuration for underwater ST.""" + + period: timedelta = pydantic.Field( + serialization_alias="period_minutes", + validation_alias="period_minutes", + gt=timedelta(), + ) + + model_config = pydantic.ConfigDict(populate_by_name=True) + + @pydantic.field_serializer("period") + def _serialize_period(self, value: timedelta, _info): + return value.total_seconds() / 60.0 + + @pydantic.field_validator("period", mode="before") + def _validate_period(cls, value: int | float | timedelta) -> timedelta: + return _validate_numeric_mins_to_timedelta(value) + + +class DrifterConfig(pydantic.BaseModel): + """Configuration for drifters.""" + + depth_meter: float = pydantic.Field(le=0.0) + lifetime: timedelta = pydantic.Field( + serialization_alias="lifetime_minutes", + validation_alias="lifetime_minutes", + gt=timedelta(), + ) + + model_config = pydantic.ConfigDict(populate_by_name=True) + + @pydantic.field_serializer("lifetime") + def _serialize_lifetime(self, value: timedelta, _info): + return value.total_seconds() / 60.0 + + @pydantic.field_validator("lifetime", mode="before") + def _validate_lifetime(cls, value: int | float | timedelta) -> timedelta: + return _validate_numeric_mins_to_timedelta(value) + + +class XBTConfig(pydantic.BaseModel): + """Configuration for xbt instrument.""" + + min_depth_meter: float = pydantic.Field(le=0.0) + max_depth_meter: float = pydantic.Field(le=0.0) + fall_speed_meter_per_second: float = pydantic.Field(gt=0.0) + deceleration_coefficient: float = pydantic.Field(gt=0.0) + + +class ShipConfig(pydantic.BaseModel): + """Configuration of the virtual ship.""" + + ship_speed_knots: float = pydantic.Field(gt=0.0) + """ + Velocity of the ship in knots. + """ + + argo_float_config: ArgoFloatConfig | None = None + """ + Argo float configuration. + + If None, no argo floats can be deployed. + """ + + adcp_config: ADCPConfig | None = None + """ + ADCP configuration. + + If None, no ADCP measurements will be performed. + """ + + ctd_config: CTDConfig | None = None + """ + CTD configuration. + + If None, no CTDs can be cast. + """ + + ctd_bgc_config: CTD_BGCConfig | None = None + """ + CTD_BGC configuration. + + If None, no BGC CTDs can be cast. + """ + + ship_underwater_st_config: ShipUnderwaterSTConfig | None = None + """ + Ship underwater salinity temperature measurementconfiguration. + + If None, no ST measurements will be performed. + """ + + drifter_config: DrifterConfig | None = None + """ + Drifter configuration. + + If None, no drifters can be deployed. + """ + + xbt_config: XBTConfig | None = None + """ + XBT configuration. + + If None, no XBTs can be cast. + """ + + model_config = pydantic.ConfigDict(extra="forbid") + + def to_yaml(self, file_path: str | Path) -> None: + """ + Write config to yaml file. + + :param file_path: Path to the file to write to. + """ + with open(file_path, "w") as file: + yaml.dump(self.model_dump(by_alias=True), file) + + @classmethod + def from_yaml(cls, file_path: str | Path) -> ShipConfig: + """ + Load config from yaml file. + + :param file_path: Path to the file to load from. + :returns: The config. + """ + with open(file_path) as file: + data = yaml.safe_load(file) + return ShipConfig(**data) + + def verify(self, schedule: Schedule) -> None: + """ + Verify the ship configuration against the provided schedule. + + This function performs two main tasks: + 1. Removes instrument configurations that are not present in the schedule. + 2. Verifies that all instruments in the schedule have corresponding configurations. + + Parameters + ---------- + schedule : Schedule + The schedule object containing the planned instruments and waypoints. + + Returns + ------- + None + + Raises + ------ + ConfigError + If an instrument in the schedule does not have a corresponding configuration. + + Notes + ----- + - Prints a message if a configuration is provided for an instrument not in the schedule. + - Sets the configuration to None for instruments not in the schedule. + - Raises a ConfigError for each instrument in the schedule that lacks a configuration. + + """ + instruments_in_schedule = schedule.get_instruments() + + for instrument in [ + "ARGO_FLOAT", + "DRIFTER", + "XBT", + "CTD", + "CTD_BGC", + ]: # TODO make instrument names consistent capitals or lowercase throughout codebase + if hasattr(self, instrument.lower() + "_config") and not any( + instrument == schedule_instrument.name + for schedule_instrument in instruments_in_schedule + ): + print(f"{instrument} configuration provided but not in schedule.") + setattr(self, instrument.lower() + "_config", None) + + # verify instruments in schedule have configuration + # TODO: the ConfigError message could be improved to explain that the **schedule** file has X instrument but the **ship_config** file does not + for instrument in instruments_in_schedule: + try: + InstrumentType(instrument) + except ValueError as e: + raise NotImplementedError("Instrument not supported.") from e + + if instrument == InstrumentType.ARGO_FLOAT and ( + not hasattr(self, "argo_float_config") or self.argo_float_config is None + ): + raise ConfigError( + "Planning has a waypoint with Argo float instrument, but configuration does not configure Argo floats." + ) + if instrument == InstrumentType.CTD and ( + not hasattr(self, "ctd_config") or self.ctd_config is None + ): + raise ConfigError( + "Planning has a waypoint with CTD instrument, but configuration does not configure CTDs." + ) + if instrument == InstrumentType.CTD_BGC and ( + not hasattr(self, "ctd_bgc_config") or self.ctd_bgc_config is None + ): + raise ConfigError( + "Planning has a waypoint with CTD_BGC instrument, but configuration does not configure CTD_BGCs." + ) + if instrument == InstrumentType.DRIFTER and ( + not hasattr(self, "drifter_config") or self.drifter_config is None + ): + raise ConfigError( + "Planning has a waypoint with drifter instrument, but configuration does not configure drifters." + ) + + if instrument == InstrumentType.XBT and ( + not hasattr(self, "xbt_config") or self.xbt_config is None + ): + raise ConfigError( + "Planning has a waypoint with XBT instrument, but configuration does not configure XBT." + ) From 851dd270c8cce9faeb8afdac6ebe0dfeda516a73 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 17 Sep 2025 15:22:46 +0100 Subject: [PATCH 07/97] add call to download_data() --- src/virtualship/cli/_fetch.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/virtualship/cli/_fetch.py b/src/virtualship/cli/_fetch.py index 127ee256a..094f8c8a3 100644 --- a/src/virtualship/cli/_fetch.py +++ b/src/virtualship/cli/_fetch.py @@ -110,14 +110,14 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None ) # keep only instruments in INTSTRUMENTS which are in schedule - filtered_instruments = { + filter_instruments = { k: v for k, v in INSTRUMENTS.items() if k in instruments_in_schedule } # iterate across instruments and download data based on space_time_region - for _, instrument in filtered_instruments.items(): + for _, instrument in filter_instruments.items(): try: - instrument["input_class"]( + input_dataset = instrument["input_class"]( data_dir=download_folder, credentials=credentials, space_time_region=space_time_region, From a417c68b350fe055e76c14cbc93ed95424787571 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 6 Oct 2025 11:32:52 +0200 Subject: [PATCH 08/97] Add new instrument classes and update InputDataset to include depth parameters --- src/virtualship/cli/_fetch.py | 5 +- src/virtualship/instruments/adcp.py | 70 ++++ src/virtualship/instruments/argo_float.py | 84 +++++ src/virtualship/instruments/ctd.py | 8 +- src/virtualship/instruments/ctd_bgc.py | 326 +++++------------- src/virtualship/instruments/drifter.py | 79 +++++ src/virtualship/instruments/master.py | 38 +- .../instruments/ship_underwater_st.py | 75 ++++ src/virtualship/instruments/xbt.py | 84 +++++ src/virtualship/models/instruments.py | 13 +- 10 files changed, 491 insertions(+), 291 deletions(-) create mode 100644 src/virtualship/instruments/adcp.py create mode 100644 src/virtualship/instruments/argo_float.py create mode 100644 src/virtualship/instruments/drifter.py create mode 100644 src/virtualship/instruments/ship_underwater_st.py create mode 100644 src/virtualship/instruments/xbt.py diff --git a/src/virtualship/cli/_fetch.py b/src/virtualship/cli/_fetch.py index 094f8c8a3..1979b1531 100644 --- a/src/virtualship/cli/_fetch.py +++ b/src/virtualship/cli/_fetch.py @@ -89,7 +89,10 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None ) shutil.copyfile(path / EXPEDITION, download_folder / EXPEDITION) - # bathymetry (required for all expeditions) + # bathymetry + # TODO: this logic means it is downloaded for all expeditions but is only needed for CTD, CTD_BGC and XBT... + # TODO: to discuss: fine to still download for all expeditions because small size and then less duplication + # TODO: or add as var in each of InputDataset objects per instrument because will be overwritten to disk anyway and therefore not duplicate? copernicusmarine.subset( dataset_id="cmems_mod_glo_phy_my_0.083deg_static", variables=["deptho"], diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py new file mode 100644 index 000000000..71d8e2af0 --- /dev/null +++ b/src/virtualship/instruments/adcp.py @@ -0,0 +1,70 @@ +from dataclasses import dataclass +from typing import ClassVar + +from virtualship.models import instruments + +## TODO: __init__.py will also need updating! +# + therefore instructions for adding new instruments will also involve adding to __init__.py as well as the new instrument script + update InstrumentType in instruments.py + + +@dataclass +class ADCP: + """ADCP configuration.""" + + name: ClassVar[str] = "ADCP" + + +# --------------- +# TODO: KERNELS +# --------------- + + +class ADCPInputDataset(instruments.InputDataset): + """Input dataset for ADCP instrument.""" + + DOWNLOAD_BUFFERS: ClassVar[dict] = { + "latlon_degrees": 0.0, + "days": 0.0, + } # ADCP data requires no buffers + + DOWNLOAD_LIMITS: ClassVar[dict] = {"min_depth": 1} + + def __init__(self, data_dir, credentials, space_time_region): + """Initialise with instrument's name.""" + super().__init__( + ADCP.name, + self.DOWNLOAD_BUFFERS["latlon_degrees"], + self.DOWNLOAD_BUFFERS["days"], + space_time_region.spatial_range.minimum_depth, + space_time_region.spatial_range.maximum_depth, + data_dir, + credentials, + space_time_region, + ) + + def get_datasets_dict(self) -> dict: + """Get variable specific args for instrument.""" + return { + "UVdata": { + "dataset_id": "cmems_mod_glo_phy-cur_anfc_0.083deg_PT6H-i", + "variables": ["uo", "vo"], + "output_filename": f"{self.name}_uv.nc", + }, + } + + +class ADCPInstrument(instruments.Instrument): + """ADCP instrument class.""" + + def __init__( + self, + config, + input_dataset, + kernels, + ): + """Initialise with instrument's name.""" + super().__init__(ADCP.name, config, input_dataset, kernels) + + def simulate(self): + """Simulate measurements.""" + ... diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py new file mode 100644 index 000000000..38eae9908 --- /dev/null +++ b/src/virtualship/instruments/argo_float.py @@ -0,0 +1,84 @@ +from dataclasses import dataclass +from datetime import timedelta +from typing import ClassVar + +from virtualship.models import Spacetime, instruments + +## TODO: __init__.py will also need updating! +# + therefore instructions for adding new instruments will also involve adding to __init__.py as well as the new instrument script + update InstrumentType in instruments.py + + +@dataclass +class ArgoFloat: + """Argo float configuration.""" + + name: ClassVar[str] = "ArgoFloat" + spacetime: Spacetime + depth: float # depth at which it floats and samples + lifetime: timedelta | None # if none, lifetime is infinite + + +# --------------- +# TODO: KERNELS +# --------------- + + +class ArgoFloatInputDataset(instruments.InputDataset): + """Input dataset for ArgoFloat instrument.""" + + DOWNLOAD_BUFFERS: ClassVar[dict] = { + "latlon_degrees": 3.0, + "days": 21.0, + } + + DOWNLOAD_LIMITS: ClassVar[dict] = {"min_depth": 1} + + def __init__(self, data_dir, credentials, space_time_region): + """Initialise with instrument's name.""" + super().__init__( + ArgoFloat.name, + self.DOWNLOAD_BUFFERS["latlon_degrees"], + self.DOWNLOAD_BUFFERS["days"], + self.DOWNLOAD_LIMITS["min_depth"], + space_time_region.spatial_range.maximum_depth, + data_dir, + credentials, + space_time_region, + ) + + def get_datasets_dict(self) -> dict: + """Get variable specific args for instrument.""" + return { + "UVdata": { + "dataset_id": "cmems_mod_glo_phy-cur_anfc_0.083deg_PT6H-i", + "variables": ["uo", "vo"], + "output_filename": "argo_float_uv.nc", + }, + "Sdata": { + "dataset_id": "cmems_mod_glo_phy-so_anfc_0.083deg_PT6H-i", + "variables": ["so"], + "output_filename": "argo_float_s.nc", + }, + "Tdata": { + "dataset_id": "cmems_mod_glo_phy-thetao_anfc_0.083deg_PT6H-i", + "variables": ["thetao"], + "output_filename": "argo_float_t.nc", + }, + } + + +class ArgoFloatInstrument(instruments.Instrument): + """ArgoFloat instrument class.""" + + def __init__( + self, + config, + input_dataset, + kernels, + ): + """Initialise with instrument's name.""" + super().__init__(ArgoFloat.name, config, input_dataset, kernels) + + def simulate(self): + """Simulate measurements.""" + ... diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 624256cb7..098132419 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -36,6 +36,8 @@ def __init__(self, data_dir, credentials, space_time_region): CTD.name, self.DOWNLOAD_BUFFERS["latlon_degrees"], self.DOWNLOAD_BUFFERS["days"], + space_time_region.spatial_range.minimum_depth, + space_time_region.spatial_range.maximum_depth, data_dir, credentials, space_time_region, @@ -72,9 +74,3 @@ def __init__( def simulate(self): """Simulate measurements.""" ... - - -# # [PSEUDO-CODE] example implementation for reference -# ctd = CTDInstrument(config=CTD, data_dir=..., kernels=...) - -# ctd.simulate(...) diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 6b9b2f298..77be7a060 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -1,263 +1,103 @@ -# from dataclasses import dataclass -# from typing import ClassVar - -# from virtualship.models import Spacetime, instruments - -# MYINSTRUMENT = "CTD_BGC" - - -# @dataclass -# class CTD_BGC: -# """CTD_BGC configuration.""" - -# spacetime: Spacetime -# min_depth: float -# max_depth: float - - -# # --------------- -# # TODO: KERNELS -# # --------------- - - -# class CTD_BGCInputDataset(instruments.InputDataset): -# """Input dataset object for CTD_BGC instrument.""" - -# DOWNLOAD_BUFFERS: ClassVar[dict] = { -# "latlon_degrees": 0.0, -# "days": 0.0, -# } # CTD_BGC data requires no buffers - -# def __init__(self, data_dir, credentials, space_time_region): -# """Initialise with instrument's name.""" -# super().__init__( -# MYINSTRUMENT, -# self.DOWNLOAD_BUFFERS["latlon_degrees"], -# self.DOWNLOAD_BUFFERS["days"], -# data_dir, -# credentials, -# space_time_region, -# ) - -# def datasets_dir(self) -> dict: -# """Variable specific args for instrument.""" -# return { -# "o2data": { -# "dataset_id": "cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m", -# "variables": ["o2"], -# "output_filename": "ctd_bgc_o2.nc", -# }, -# "chlorodata": { -# "dataset_id": "cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m", -# "variables": ["chl"], -# "output_filename": "ctd_bgc_chloro.nc", -# }, -# } - - -# class CTD_BGCInstrument(instruments.Instrument): -# """CTD_BGC instrument class.""" - -# def __init__( -# self, -# config, -# input_dataset, -# kernels, -# ): -# """Initialise with instrument's name.""" -# super().__init__(MYINSTRUMENT, config, input_dataset, kernels) - -# def simulate(self): -# """Simulate measurements.""" -# ... - - -# # # [PSEUDO-CODE] example implementation for reference -# # ctd = CTD_BGCInstrument(config=CTD_BGC, data_dir=..., kernels=...) - -# # ctd.simulate(...) - - -"""CTD_BGC instrument.""" - from dataclasses import dataclass -from datetime import timedelta -from pathlib import Path +from typing import ClassVar -import numpy as np -from parcels import FieldSet, JITParticle, ParticleSet, Variable - -from virtualship.models import Spacetime +from virtualship.models import Spacetime, instruments @dataclass class CTD_BGC: - """Configuration for a single BGC CTD.""" + """CTD_BGC configuration.""" + name: ClassVar[str] = "CTD_BGC" spacetime: Spacetime min_depth: float max_depth: float -_CTD_BGCParticle = JITParticle.add_variables( - [ - Variable("o2", dtype=np.float32, initial=np.nan), - Variable("chl", dtype=np.float32, initial=np.nan), - Variable("no3", dtype=np.float32, initial=np.nan), - Variable("po4", dtype=np.float32, initial=np.nan), - Variable("ph", dtype=np.float32, initial=np.nan), - Variable("phyc", dtype=np.float32, initial=np.nan), - Variable("zooc", dtype=np.float32, initial=np.nan), - Variable("nppv", dtype=np.float32, initial=np.nan), - Variable("raising", dtype=np.int8, initial=0.0), # bool. 0 is False, 1 is True. - Variable("max_depth", dtype=np.float32), - Variable("min_depth", dtype=np.float32), - Variable("winch_speed", dtype=np.float32), - ] -) - - -def _sample_o2(particle, fieldset, time): - particle.o2 = fieldset.o2[time, particle.depth, particle.lat, particle.lon] +# --------------- +# TODO: KERNELS +# --------------- -def _sample_chlorophyll(particle, fieldset, time): - particle.chl = fieldset.chl[time, particle.depth, particle.lat, particle.lon] +class CTD_BGCInputDataset(instruments.InputDataset): + """Input dataset object for CTD_BGC instrument.""" + DOWNLOAD_BUFFERS: ClassVar[dict] = { + "latlon_degrees": 0.0, + "days": 0.0, + } # CTD_BGC data requires no buffers -def _sample_nitrate(particle, fieldset, time): - particle.no3 = fieldset.no3[time, particle.depth, particle.lat, particle.lon] - - -def _sample_phosphate(particle, fieldset, time): - particle.po4 = fieldset.po4[time, particle.depth, particle.lat, particle.lon] - - -def _sample_ph(particle, fieldset, time): - particle.ph = fieldset.ph[time, particle.depth, particle.lat, particle.lon] - - -def _sample_phytoplankton(particle, fieldset, time): - particle.phyc = fieldset.phyc[time, particle.depth, particle.lat, particle.lon] - - -def _sample_zooplankton(particle, fieldset, time): - particle.zooc = fieldset.zooc[time, particle.depth, particle.lat, particle.lon] - - -def _sample_primary_production(particle, fieldset, time): - particle.nppv = fieldset.nppv[time, particle.depth, particle.lat, particle.lon] - - -def _ctd_bgc_cast(particle, fieldset, time): - # lowering - if particle.raising == 0: - particle_ddepth = -particle.winch_speed * particle.dt - if particle.depth + particle_ddepth < particle.max_depth: - particle.raising = 1 - particle_ddepth = -particle_ddepth - # raising - else: - particle_ddepth = particle.winch_speed * particle.dt - if particle.depth + particle_ddepth > particle.min_depth: - particle.delete() - - -def simulate_ctd_bgc( - fieldset: FieldSet, - out_path: str | Path, - ctd_bgcs: list[CTD_BGC], - outputdt: timedelta, -) -> None: - """ - Use Parcels to simulate a set of BGC CTDs in a fieldset. - - :param fieldset: The fieldset to simulate the BGC CTDs in. - :param out_path: The path to write the results to. - :param ctds: A list of BGC CTDs to simulate. - :param outputdt: Interval which dictates the update frequency of file output during simulation - :raises ValueError: Whenever provided BGC CTDs, fieldset, are not compatible with this function. - """ - WINCH_SPEED = 1.0 # sink and rise speed in m/s - DT = 10.0 # dt of CTD simulation integrator - - if len(ctd_bgcs) == 0: - print( - "No BGC CTDs provided. Parcels currently crashes when providing an empty particle set, so no BGC CTD simulation will be done and no files will be created." + def __init__(self, data_dir, credentials, space_time_region): + """Initialise with instrument's name.""" + super().__init__( + CTD_BGC.name, + self.DOWNLOAD_BUFFERS["latlon_degrees"], + self.DOWNLOAD_BUFFERS["days"], + space_time_region.spatial_range.minimum_depth, + space_time_region.spatial_range.maximum_depth, + data_dir, + credentials, + space_time_region, ) - # TODO when Parcels supports it this check can be removed. - return - fieldset_starttime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[0]) - fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) - - # deploy time for all ctds should be later than fieldset start time - if not all( - [ - np.datetime64(ctd_bgc.spacetime.time) >= fieldset_starttime - for ctd_bgc in ctd_bgcs - ] + def datasets_dir(self) -> dict: + """Variable specific args for instrument.""" + return { + "o2data": { + "dataset_id": "cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m", + "variables": ["o2"], + "output_filename": "ctd_bgc_o2.nc", + }, + "chlorodata": { + "dataset_id": "cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m", + "variables": ["chl"], + "output_filename": "ctd_bgc_chl.nc", + }, + "nitratedata": { + "dataset_id": "cmems_mod_glo_bgc-nut_anfc_0.25deg_P1D-m", + "variables": ["no3"], + "output_filename": "ctd_bgc_no3.nc", + }, + "phosphatedata": { + "dataset_id": "cmems_mod_glo_bgc-nut_anfc_0.25deg_P1D-m", + "variables": ["po4"], + "output_filename": "ctd_bgc_po4.nc", + }, + "phdata": { + "dataset_id": "cmems_mod_glo_bgc-car_anfc_0.25deg_P1D-m", + "variables": ["ph"], + "output_filename": "ctd_bgc_ph.nc", + }, + "phytoplanktondata": { + "dataset_id": "cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m", + "variables": ["phyc"], + "output_filename": "ctd_bgc_phyc.nc", + }, + "zooplanktondata": { + "dataset_id": "cmems_mod_glo_bgc-plankton_anfc_0.25deg_P1D-m", + "variables": ["zooc"], + "output_filename": "ctd_bgc_zooc.nc", + }, + "primaryproductiondata": { + "dataset_id": "cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m", + "variables": ["nppv"], + "output_filename": "ctd_bgc_nppv.nc", + }, + } + + +class CTD_BGCInstrument(instruments.Instrument): + """CTD_BGC instrument class.""" + + def __init__( + self, + config, + input_dataset, + kernels, ): - raise ValueError("BGC CTD deployed before fieldset starts.") - - # depth the bgc ctd will go to. shallowest between bgc ctd max depth and bathymetry. - max_depths = [ - max( - ctd_bgc.max_depth, - fieldset.bathymetry.eval( - z=0, - y=ctd_bgc.spacetime.location.lat, - x=ctd_bgc.spacetime.location.lon, - time=0, - ), - ) - for ctd_bgc in ctd_bgcs - ] + """Initialise with instrument's name.""" + super().__init__(CTD_BGC.name, config, input_dataset, kernels) - # CTD depth can not be too shallow, because kernel would break. - # This shallow is not useful anyway, no need to support. - if not all([max_depth <= -DT * WINCH_SPEED for max_depth in max_depths]): - raise ValueError( - f"BGC CTD max_depth or bathymetry shallower than maximum {-DT * WINCH_SPEED}" - ) - - # define parcel particles - ctd_bgc_particleset = ParticleSet( - fieldset=fieldset, - pclass=_CTD_BGCParticle, - lon=[ctd_bgc.spacetime.location.lon for ctd_bgc in ctd_bgcs], - lat=[ctd_bgc.spacetime.location.lat for ctd_bgc in ctd_bgcs], - depth=[ctd_bgc.min_depth for ctd_bgc in ctd_bgcs], - time=[ctd_bgc.spacetime.time for ctd_bgc in ctd_bgcs], - max_depth=max_depths, - min_depth=[ctd_bgc.min_depth for ctd_bgc in ctd_bgcs], - winch_speed=[WINCH_SPEED for _ in ctd_bgcs], - ) - - # define output file for the simulation - out_file = ctd_bgc_particleset.ParticleFile(name=out_path, outputdt=outputdt) - - # execute simulation - ctd_bgc_particleset.execute( - [ - _sample_o2, - _sample_chlorophyll, - _sample_nitrate, - _sample_phosphate, - _sample_ph, - _sample_phytoplankton, - _sample_zooplankton, - _sample_primary_production, - _ctd_bgc_cast, - ], - endtime=fieldset_endtime, - dt=DT, - verbose_progress=False, - output_file=out_file, - ) - - # there should be no particles left, as they delete themselves when they resurface - if len(ctd_bgc_particleset.particledata) != 0: - raise ValueError( - "Simulation ended before BGC CTD resurfaced. This most likely means the field time dimension did not match the simulation time span." - ) + def simulate(self): + """Simulate measurements.""" + ... diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py new file mode 100644 index 000000000..70ac4b7bb --- /dev/null +++ b/src/virtualship/instruments/drifter.py @@ -0,0 +1,79 @@ +from dataclasses import dataclass +from datetime import timedelta +from typing import ClassVar + +from virtualship.models import Spacetime, instruments + +## TODO: __init__.py will also need updating! +# + therefore instructions for adding new instruments will also involve adding to __init__.py as well as the new instrument script + update InstrumentType in instruments.py + + +@dataclass +class Drifter: + """Drifter configuration.""" + + name: ClassVar[str] = "Drifter" + spacetime: Spacetime + depth: float # depth at which it floats and samples + lifetime: timedelta | None # if none, lifetime is infinite + + +# --------------- +# TODO: KERNELS +# --------------- + + +class DrifterInputDataset(instruments.InputDataset): + """Input dataset for Drifter instrument.""" + + DOWNLOAD_BUFFERS: ClassVar[dict] = { + "latlon_degrees": 3.0, + "days": 21.0, + } + + DOWNLOAD_LIMITS: ClassVar[dict] = {"min_depth": 1, "max_depth": 1} + + def __init__(self, data_dir, credentials, space_time_region): + """Initialise with instrument's name.""" + super().__init__( + Drifter.name, + self.DOWNLOAD_BUFFERS["latlon_degrees"], + self.DOWNLOAD_BUFFERS["days"], + self.DOWNLOAD_LIMITS["min_depth"], + self.DOWNLOAD_LIMITS["max_depth"], + data_dir, + credentials, + space_time_region, + ) + + def get_datasets_dict(self) -> dict: + """Get variable specific args for instrument.""" + return { + "UVdata": { + "dataset_id": "cmems_mod_glo_phy-cur_anfc_0.083deg_PT6H-i", + "variables": ["uo", "vo"], + "output_filename": "drifter_uv.nc", + }, + "Tdata": { + "dataset_id": "cmems_mod_glo_phy-thetao_anfc_0.083deg_PT6H-i", + "variables": ["thetao"], + "output_filename": "drifter_t.nc", + }, + } + + +class DrifterInstrument(instruments.Instrument): + """Drifter instrument class.""" + + def __init__( + self, + config, + input_dataset, + kernels, + ): + """Initialise with instrument's name.""" + super().__init__(Drifter.name, config, input_dataset, kernels) + + def simulate(self): + """Simulate measurements.""" + ... diff --git a/src/virtualship/instruments/master.py b/src/virtualship/instruments/master.py index 4705c8e3a..161348e2a 100644 --- a/src/virtualship/instruments/master.py +++ b/src/virtualship/instruments/master.py @@ -15,29 +15,16 @@ class InstrumentType(Enum): """Types of the instruments.""" CTD = "CTD" - # CTD_BGC = "CTD_BGC" - # DRIFTER = "DRIFTER" - # ARGO_FLOAT = "ARGO_FLOAT" - # XBT = "XBT" + CTD_BGC = "CTD_BGC" + DRIFTER = "DRIFTER" + ARGO_FLOAT = "ARGO_FLOAT" + XBT = "XBT" # # TODO: should underway also be handled here?! # ADCP = "ADCP" # UNDERWAY_ST = "UNDERWAY_ST" -# replace with imports instead... -class CTDInputDataset: - """Input dataset class for CTD instrument.""" - - pass - - -class CTDInstrument: - """Instrument class for CTD instrument.""" - - pass - - INSTRUMENTS = { inst: { "input_class": globals()[f"{inst.value}InputDataset"], @@ -47,20 +34,3 @@ class CTDInstrument: if f"{inst.value}InputDataset" in globals() and f"{inst.value}Instrument" in globals() } - - -# INSTRUMENTS = { -# InstrumentType.CTD: { -# "input_class": CTDInputDataset, -# "instrument_class": CTDInstrument, -# } -# # and so on for other instruments... -# } - -# INSTRUMENTS = { -# "InstrumentType.CTD": { -# "input_class": "test", -# "instrument_class": "test", -# } -# # and so on for other instruments... -# } diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py new file mode 100644 index 000000000..f364b1560 --- /dev/null +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -0,0 +1,75 @@ +from dataclasses import dataclass +from typing import ClassVar + +from virtualship.models import instruments + +## TODO: __init__.py will also need updating! +# + therefore instructions for adding new instruments will also involve adding to __init__.py as well as the new instrument script + update InstrumentType in instruments.py + + +@dataclass +class Underwater_ST: + """Underwater_ST configuration.""" + + name: ClassVar[str] = "Underwater_ST" + + +# --------------- +# TODO: KERNELS +# --------------- + + +class Underwater_STInputDataset(instruments.InputDataset): + """Input dataset for Underwater_ST instrument.""" + + DOWNLOAD_BUFFERS: ClassVar[dict] = { + "latlon_degrees": 0.0, + "days": 0.0, + } # Underwater_ST data requires no buffers + + DOWNLOAD_LIMITS: ClassVar[dict] = {"min_depth": 1} + + def __init__(self, data_dir, credentials, space_time_region): + """Initialise with instrument's name.""" + super().__init__( + Underwater_ST.name, + self.DOWNLOAD_BUFFERS["latlon_degrees"], + self.DOWNLOAD_BUFFERS["days"], + -2.0, # is always at 2m depth + -2.0, # is always at 2m depth + data_dir, + credentials, + space_time_region, + ) + + def get_datasets_dict(self) -> dict: + """Get variable specific args for instrument.""" + return { + "Sdata": { + "dataset_id": "cmems_mod_glo_phy-so_anfc_0.083deg_PT6H-i", + "variables": ["so"], + "output_filename": f"{self.name}_s.nc", + }, + "Tdata": { + "dataset_id": "cmems_mod_glo_phy-thetao_anfc_0.083deg_PT6H-i", + "variables": ["thetao"], + "output_filename": f"{self.name}_t.nc", + }, + } + + +class Underwater_STInstrument(instruments.Instrument): + """Underwater_ST instrument class.""" + + def __init__( + self, + config, + input_dataset, + kernels, + ): + """Initialise with instrument's name.""" + super().__init__(Underwater_ST.name, config, input_dataset, kernels) + + def simulate(self): + """Simulate measurements.""" + ... diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py new file mode 100644 index 000000000..dcec018b4 --- /dev/null +++ b/src/virtualship/instruments/xbt.py @@ -0,0 +1,84 @@ +from dataclasses import dataclass +from datetime import timedelta +from typing import ClassVar + +from virtualship.models import Spacetime, instruments + +## TODO: __init__.py will also need updating! +# + therefore instructions for adding new instruments will also involve adding to __init__.py as well as the new instrument script + update InstrumentType in instruments.py + + +@dataclass +class XBT: + """XBT configuration.""" + + name: ClassVar[str] = "XBT" + spacetime: Spacetime + depth: float # depth at which it floats and samples + lifetime: timedelta | None # if none, lifetime is infinite + + +# --------------- +# TODO: KERNELS +# --------------- + + +class XBTInputDataset(instruments.InputDataset): + """Input dataset for XBT instrument.""" + + DOWNLOAD_BUFFERS: ClassVar[dict] = { + "latlon_degrees": 3.0, + "days": 21.0, + } + + DOWNLOAD_LIMITS: ClassVar[dict] = {"min_depth": 1} + + def __init__(self, data_dir, credentials, space_time_region): + """Initialise with instrument's name.""" + super().__init__( + XBT.name, + self.DOWNLOAD_BUFFERS["latlon_degrees"], + self.DOWNLOAD_BUFFERS["days"], + self.DOWNLOAD_LIMITS["min_depth"], + space_time_region.spatial_range.maximum_depth, + data_dir, + credentials, + space_time_region, + ) + + def get_datasets_dict(self) -> dict: + """Get variable specific args for instrument.""" + return { + "UVdata": { + "dataset_id": "cmems_mod_glo_phy-cur_anfc_0.083deg_PT6H-i", + "variables": ["uo", "vo"], + "output_filename": "ship_uv.nc", + }, + "Sdata": { + "dataset_id": "cmems_mod_glo_phy-so_anfc_0.083deg_PT6H-i", + "variables": ["so"], + "output_filename": "ship_s.nc", + }, + "Tdata": { + "dataset_id": "cmems_mod_glo_phy-thetao_anfc_0.083deg_PT6H-i", + "variables": ["thetao"], + "output_filename": "ship_t.nc", + }, + } + + +class XBTInstrument(instruments.Instrument): + """XBT instrument class.""" + + def __init__( + self, + config, + input_dataset, + kernels, + ): + """Initialise with instrument's name.""" + super().__init__(XBT.name, config, input_dataset, kernels) + + def simulate(self): + """Simulate measurements.""" + ... diff --git a/src/virtualship/models/instruments.py b/src/virtualship/models/instruments.py index cddebd85f..57139add2 100644 --- a/src/virtualship/models/instruments.py +++ b/src/virtualship/models/instruments.py @@ -31,6 +31,8 @@ def __init__( name: str, latlon_buffer: float, datetime_buffer: float, + min_depth: float, + max_depth: float, data_dir: str, credentials: dict, space_time_region: SpaceTimeRegion, @@ -39,6 +41,8 @@ def __init__( self.name = name self.latlon_buffer = latlon_buffer self.datetime_buffer = datetime_buffer + self.min_depth = min_depth + self.max_depth = max_depth self.data_dir = data_dir self.credentials = credentials self.space_time_region = space_time_region @@ -62,8 +66,8 @@ def download_data(self) -> None: start_datetime=self.space_time_region.time_range.start_time, end_datetime=self.space_time_region.time_range.end_time + timedelta(days=self.datetime_buffer), - minimum_depth=abs(self.space_time_region.spatial_range.minimum_depth), - maximum_depth=abs(self.space_time_region.spatial_range.maximum_depth), + minimum_depth=abs(self.min_depth), + maximum_depth=abs(self.max_depth), output_directory=self.data_dir, username=self.credentials["username"], password=self.credentials["password"], @@ -77,11 +81,6 @@ def download_data(self) -> None: download_args = {**parameter_args, **dataset} copernicusmarine.subset(**download_args) - # def get_fieldset_paths(self) -> list: - # """List of paths for instrument's (downloaded) input data.""" - - # ... - class Instrument(abc.ABC): """Base class for instruments.""" From a0c77abe91ae43f32889a39a1c26b0bfd0d178ca Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 7 Oct 2025 11:52:28 +0200 Subject: [PATCH 09/97] Refactor instrument handling in _fetch and update imports for consistency --- src/virtualship/cli/_fetch.py | 21 +++++++++- src/virtualship/instruments/master.py | 56 ++++++++++++++++----------- src/virtualship/models/instruments.py | 7 ---- src/virtualship/models/schedule.py | 2 +- src/virtualship/models/ship_config.py | 2 +- 5 files changed, 54 insertions(+), 34 deletions(-) diff --git a/src/virtualship/cli/_fetch.py b/src/virtualship/cli/_fetch.py index 1979b1531..c258f0446 100644 --- a/src/virtualship/cli/_fetch.py +++ b/src/virtualship/cli/_fetch.py @@ -25,9 +25,12 @@ import virtualship.cli._creds as creds from virtualship.utils import EXPEDITION from virtualship.instruments.master import INSTRUMENTS +from virtualship.instruments.master import InstrumentType, get_instruments_registry DOWNLOAD_METADATA = "download_metadata.yaml" +INSTRUMENTS = get_instruments_registry() + def _fetch(path: str | Path, username: str | None, password: str | None) -> None: """ @@ -81,6 +84,20 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None end_datetime = time_range.end_time instruments_in_schedule = expedition.schedule.get_instruments() + # TEMPORARY measure to get underway instruments in `instruments_in_schedule` + # TODO: should evaporate when schedule and ship_config.yaml files are consolidated in a separate PR... + if ship_config.adcp_config is not None: + instruments_in_schedule.add(InstrumentType.ADCP) + if ship_config.ship_underwater_st_config is not None: + instruments_in_schedule.add(InstrumentType.UNDERWATER_ST) + + # TEMPORARY measure to get underway instruments in `instruments_in_schedule` + # TODO: should evaporate when schedule and ship_config.yaml files are consolidated in a separate PR... + if ship_config.adcp_config is not None: + instruments_in_schedule.add(InstrumentType.ADCP) + if ship_config.ship_underwater_st_config is not None: + instruments_in_schedule.add(InstrumentType.UNDERWATER_ST) + # Create download folder and set download metadata download_folder = data_dir / hash_to_filename(space_time_region_hash) download_folder.mkdir() @@ -118,7 +135,7 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None } # iterate across instruments and download data based on space_time_region - for _, instrument in filter_instruments.items(): + for itype, instrument in filter_instruments.items(): try: input_dataset = instrument["input_class"]( data_dir=download_folder, @@ -237,7 +254,7 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None shutil.rmtree(download_folder) raise e - click.echo(f"{instrument.name} data download completed.") # TODO + click.echo(f"{itype.value} data download completed.") complete_download(download_folder) diff --git a/src/virtualship/instruments/master.py b/src/virtualship/instruments/master.py index 161348e2a..58eb05915 100644 --- a/src/virtualship/instruments/master.py +++ b/src/virtualship/instruments/master.py @@ -1,36 +1,46 @@ -# - -# TODO: temporary measure so as not to have to overhaul the InstrumentType class logic in one go -#! And also to avoid breaking other parts of the codebase which rely on InstrumentType when for now just working on fetch -# TODO: ideally this can evaporate... -# TODO: discuss to see if there's a better option...! - from enum import Enum -# and so on ... -# from virtualship.instruments.ctd import CTDInputDataset, CTDInstrument - class InstrumentType(Enum): """Types of the instruments.""" + # TODO: temporary measure so as not to have to overhaul the InstrumentType class logic in one go + #! And also to avoid breaking other parts of the codebase which rely on InstrumentType when for now just working on fetch + # TODO: ideally this can evaporate in a future PR... + CTD = "CTD" CTD_BGC = "CTD_BGC" DRIFTER = "DRIFTER" ARGO_FLOAT = "ARGO_FLOAT" XBT = "XBT" + ADCP = "ADCP" + UNDERWATER_ST = "UNDERWATER_ST" + + +def get_instruments_registry(): + # local imports to avoid circular import issues + from virtualship.instruments.adcp import ADCPInputDataset + from virtualship.instruments.argo_float import ArgoFloatInputDataset + from virtualship.instruments.ctd import CTDInputDataset + from virtualship.instruments.ctd_bgc import CTD_BGCInputDataset + from virtualship.instruments.drifter import DrifterInputDataset + from virtualship.instruments.ship_underwater_st import Underwater_STInputDataset + from virtualship.instruments.xbt import XBTInputDataset + + _input_class_map = { + "CTD": CTDInputDataset, + "CTD_BGC": CTD_BGCInputDataset, + "DRIFTER": DrifterInputDataset, + "ARGO_FLOAT": ArgoFloatInputDataset, + "XBT": XBTInputDataset, + "ADCP": ADCPInputDataset, + "UNDERWATER_ST": Underwater_STInputDataset, + } - # # TODO: should underway also be handled here?! - # ADCP = "ADCP" - # UNDERWAY_ST = "UNDERWAY_ST" - - -INSTRUMENTS = { - inst: { - "input_class": globals()[f"{inst.value}InputDataset"], - "instrument_class": globals()[f"{inst.value}Instrument"], + return { + inst: { + "input_class": _input_class_map.get(inst.value), + } + for inst in InstrumentType + if _input_class_map.get(inst.value) is not None } - for inst in InstrumentType - if f"{inst.value}InputDataset" in globals() - and f"{inst.value}Instrument" in globals() -} diff --git a/src/virtualship/models/instruments.py b/src/virtualship/models/instruments.py index 57139add2..7a3562908 100644 --- a/src/virtualship/models/instruments.py +++ b/src/virtualship/models/instruments.py @@ -121,10 +121,3 @@ def run(self): def simulate(self): """Simulate instrument measurements.""" ... - - -# e.g. pseudo-code ... -# TODO: (necessary?) how to dynamically assemble list of all instruments defined so that new instruments can be added only by changes in one place...? -available_instruments: list = ... -# for instrument in available_instruments: -# MyInstrument(instrument) diff --git a/src/virtualship/models/schedule.py b/src/virtualship/models/schedule.py index 091c23b49..f3e5dabea 100644 --- a/src/virtualship/models/schedule.py +++ b/src/virtualship/models/schedule.py @@ -12,8 +12,8 @@ import yaml from virtualship.errors import ScheduleError +from virtualship.instruments.master import InstrumentType -from .instruments import InstrumentType from .location import Location from .space_time_region import SpaceTimeRegion diff --git a/src/virtualship/models/ship_config.py b/src/virtualship/models/ship_config.py index 61d3d3905..ba7d221f0 100644 --- a/src/virtualship/models/ship_config.py +++ b/src/virtualship/models/ship_config.py @@ -10,7 +10,7 @@ import yaml from virtualship.errors import ConfigError -from virtualship.models.instruments import InstrumentType +from virtualship.instruments.master import InstrumentType from virtualship.utils import _validate_numeric_mins_to_timedelta if TYPE_CHECKING: From 9bcaba2864566b02b5f8ec4ceeba463b2152aba0 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 7 Oct 2025 11:53:34 +0200 Subject: [PATCH 10/97] Refactor instrument classes and re-add (temporary) simulation functions across multiple files --- src/virtualship/instruments/adcp.py | 111 ++++++++-- src/virtualship/instruments/argo_float.py | 200 ++++++++++++++++-- src/virtualship/instruments/ctd.py | 148 +++++++++++-- src/virtualship/instruments/ctd_bgc.py | 196 +++++++++++++++-- src/virtualship/instruments/drifter.py | 129 +++++++++-- .../instruments/ship_underwater_st.py | 102 +++++++-- src/virtualship/instruments/xbt.py | 151 +++++++++++-- 7 files changed, 917 insertions(+), 120 deletions(-) diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index 71d8e2af0..052004fba 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -1,7 +1,11 @@ from dataclasses import dataclass +from pathlib import Path from typing import ClassVar -from virtualship.models import instruments +import numpy as np +from parcels import FieldSet, ParticleSet, ScipyParticle, Variable + +from virtualship.models import Spacetime, instruments ## TODO: __init__.py will also need updating! # + therefore instructions for adding new instruments will also involve adding to __init__.py as well as the new instrument script + update InstrumentType in instruments.py @@ -14,9 +18,20 @@ class ADCP: name: ClassVar[str] = "ADCP" -# --------------- -# TODO: KERNELS -# --------------- +# we specifically use ScipyParticle because we have many small calls to execute +# there is some overhead with JITParticle and this ends up being significantly faster +_ADCPParticle = ScipyParticle.add_variables( + [ + Variable("U", dtype=np.float32, initial=np.nan), + Variable("V", dtype=np.float32, initial=np.nan), + ] +) + + +def _sample_velocity(particle, fieldset, time): + particle.U, particle.V = fieldset.UV.eval( + time, particle.depth, particle.lat, particle.lon, applyConversion=False + ) class ADCPInputDataset(instruments.InputDataset): @@ -53,18 +68,78 @@ def get_datasets_dict(self) -> dict: } -class ADCPInstrument(instruments.Instrument): - """ADCP instrument class.""" - - def __init__( - self, - config, - input_dataset, - kernels, - ): - """Initialise with instrument's name.""" - super().__init__(ADCP.name, config, input_dataset, kernels) +# TODO: uncomment when ready for new simulation logic! +# class ADCPInstrument(instruments.Instrument): +# """ADCP instrument class.""" + +# def __init__( +# self, +# config, +# input_dataset, +# kernels, +# ): +# """Initialise with instrument's name.""" +# super().__init__(ADCP.name, config, input_dataset, kernels) + +# def simulate(self): +# """Simulate measurements.""" +# ... + + +# TODO: to be replaced with new simulation logic +## -- old simulation code + + +def simulate_adcp( + fieldset: FieldSet, + out_path: str | Path, + max_depth: float, + min_depth: float, + num_bins: int, + sample_points: list[Spacetime], +) -> None: + """ + Use Parcels to simulate an ADCP in a fieldset. + + :param fieldset: The fieldset to simulate the ADCP in. + :param out_path: The path to write the results to. + :param max_depth: Maximum depth the ADCP can measure. + :param min_depth: Minimum depth the ADCP can measure. + :param num_bins: How many samples to take in the complete range between max_depth and min_depth. + :param sample_points: The places and times to sample at. + """ + sample_points.sort(key=lambda p: p.time) + + bins = np.linspace(max_depth, min_depth, num_bins) + num_particles = len(bins) + particleset = ParticleSet.from_list( + fieldset=fieldset, + pclass=_ADCPParticle, + lon=np.full( + num_particles, 0.0 + ), # initial lat/lon are irrelevant and will be overruled later. + lat=np.full(num_particles, 0.0), + depth=bins, + time=0, # same for time + ) + + # define output file for the simulation + # outputdt set to infinite as we just want to write at the end of every call to 'execute' + out_file = particleset.ParticleFile(name=out_path, outputdt=np.inf) + + for point in sample_points: + particleset.lon_nextloop[:] = point.location.lon + particleset.lat_nextloop[:] = point.location.lat + particleset.time_nextloop[:] = fieldset.time_origin.reltime( + np.datetime64(point.time) + ) - def simulate(self): - """Simulate measurements.""" - ... + # perform one step using the particleset + # dt and runtime are set so exactly one step is made. + particleset.execute( + [_sample_velocity], + dt=1, + runtime=1, + verbose_progress=False, + output_file=out_file, + ) diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index 38eae9908..4769eccfa 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -1,7 +1,19 @@ +import math from dataclasses import dataclass -from datetime import timedelta +from datetime import datetime, timedelta +from pathlib import Path from typing import ClassVar +import numpy as np +from parcels import ( + AdvectionRK4, + FieldSet, + JITParticle, + ParticleSet, + StatusCode, + Variable, +) + from virtualship.models import Spacetime, instruments ## TODO: __init__.py will also need updating! @@ -18,9 +30,88 @@ class ArgoFloat: lifetime: timedelta | None # if none, lifetime is infinite -# --------------- -# TODO: KERNELS -# --------------- +_ArgoParticle = JITParticle.add_variables( + [ + Variable("cycle_phase", dtype=np.int32, initial=0.0), + Variable("cycle_age", dtype=np.float32, initial=0.0), + Variable("drift_age", dtype=np.float32, initial=0.0), + Variable("salinity", dtype=np.float32, initial=np.nan), + Variable("temperature", dtype=np.float32, initial=np.nan), + Variable("min_depth", dtype=np.float32), + Variable("max_depth", dtype=np.float32), + Variable("drift_depth", dtype=np.float32), + Variable("vertical_speed", dtype=np.float32), + Variable("cycle_days", dtype=np.int32), + Variable("drift_days", dtype=np.int32), + ] +) + + +def _argo_float_vertical_movement(particle, fieldset, time): + if particle.cycle_phase == 0: + # Phase 0: Sinking with vertical_speed until depth is drift_depth + particle_ddepth += ( # noqa Parcels defines particle_* variables, which code checkers cannot know. + particle.vertical_speed * particle.dt + ) + if particle.depth + particle_ddepth <= particle.drift_depth: + particle_ddepth = particle.drift_depth - particle.depth + particle.cycle_phase = 1 + + elif particle.cycle_phase == 1: + # Phase 1: Drifting at depth for drifttime seconds + particle.drift_age += particle.dt + if particle.drift_age >= particle.drift_days * 86400: + particle.drift_age = 0 # reset drift_age for next cycle + particle.cycle_phase = 2 + + elif particle.cycle_phase == 2: + # Phase 2: Sinking further to max_depth + particle_ddepth += particle.vertical_speed * particle.dt + if particle.depth + particle_ddepth <= particle.max_depth: + particle_ddepth = particle.max_depth - particle.depth + particle.cycle_phase = 3 + + elif particle.cycle_phase == 3: + # Phase 3: Rising with vertical_speed until at surface + particle_ddepth -= particle.vertical_speed * particle.dt + particle.cycle_age += ( + particle.dt + ) # solve issue of not updating cycle_age during ascent + if particle.depth + particle_ddepth >= particle.min_depth: + particle_ddepth = particle.min_depth - particle.depth + particle.temperature = ( + math.nan + ) # reset temperature to NaN at end of sampling cycle + particle.salinity = math.nan # idem + particle.cycle_phase = 4 + else: + particle.temperature = fieldset.T[ + time, particle.depth, particle.lat, particle.lon + ] + particle.salinity = fieldset.S[ + time, particle.depth, particle.lat, particle.lon + ] + + elif particle.cycle_phase == 4: + # Phase 4: Transmitting at surface until cycletime is reached + if particle.cycle_age > particle.cycle_days * 86400: + particle.cycle_phase = 0 + particle.cycle_age = 0 + + if particle.state == StatusCode.Evaluate: + particle.cycle_age += particle.dt # update cycle_age + + +def _keep_at_surface(particle, fieldset, time): + # Prevent error when float reaches surface + if particle.state == StatusCode.ErrorThroughSurface: + particle.depth = particle.min_depth + particle.state = StatusCode.Success + + +def _check_error(particle, fieldset, time): + if particle.state >= 50: # This captures all Errors + particle.delete() class ArgoFloatInputDataset(instruments.InputDataset): @@ -67,18 +158,89 @@ def get_datasets_dict(self) -> dict: } -class ArgoFloatInstrument(instruments.Instrument): - """ArgoFloat instrument class.""" - - def __init__( - self, - config, - input_dataset, - kernels, - ): - """Initialise with instrument's name.""" - super().__init__(ArgoFloat.name, config, input_dataset, kernels) - - def simulate(self): - """Simulate measurements.""" - ... +# class ArgoFloatInstrument(instruments.Instrument): +# """ArgoFloat instrument class.""" + +# def __init__( +# self, +# config, +# input_dataset, +# kernels, +# ): +# """Initialise with instrument's name.""" +# super().__init__(ArgoFloat.name, config, input_dataset, kernels) + +# def simulate(self): +# """Simulate measurements.""" +# ... + + +def simulate_argo_floats( + fieldset: FieldSet, + out_path: str | Path, + argo_floats: list[ArgoFloat], + outputdt: timedelta, + endtime: datetime | None, +) -> None: + """ + Use Parcels to simulate a set of Argo floats in a fieldset. + + :param fieldset: The fieldset to simulate the Argo floats in. + :param out_path: The path to write the results to. + :param argo_floats: A list of Argo floats to simulate. + :param outputdt: Interval which dictates the update frequency of file output during simulation + :param endtime: Stop at this time, or if None, continue until the end of the fieldset. + """ + DT = 10.0 # dt of Argo float simulation integrator + + if len(argo_floats) == 0: + print( + "No Argo floats provided. Parcels currently crashes when providing an empty particle set, so no argo floats simulation will be done and no files will be created." + ) + # TODO when Parcels supports it this check can be removed. + return + + # define parcel particles + argo_float_particleset = ParticleSet( + fieldset=fieldset, + pclass=_ArgoParticle, + lat=[argo.spacetime.location.lat for argo in argo_floats], + lon=[argo.spacetime.location.lon for argo in argo_floats], + depth=[argo.min_depth for argo in argo_floats], + time=[argo.spacetime.time for argo in argo_floats], + min_depth=[argo.min_depth for argo in argo_floats], + max_depth=[argo.max_depth for argo in argo_floats], + drift_depth=[argo.drift_depth for argo in argo_floats], + vertical_speed=[argo.vertical_speed for argo in argo_floats], + cycle_days=[argo.cycle_days for argo in argo_floats], + drift_days=[argo.drift_days for argo in argo_floats], + ) + + # define output file for the simulation + out_file = argo_float_particleset.ParticleFile( + name=out_path, outputdt=outputdt, chunks=[len(argo_float_particleset), 100] + ) + + # get earliest between fieldset end time and provide end time + fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) + if endtime is None: + actual_endtime = fieldset_endtime + elif endtime > fieldset_endtime: + print("WARN: Requested end time later than fieldset end time.") + actual_endtime = fieldset_endtime + else: + actual_endtime = np.timedelta64(endtime) + + # execute simulation + argo_float_particleset.execute( + [ + _argo_float_vertical_movement, + AdvectionRK4, + _keep_at_surface, + _check_error, + ], + endtime=actual_endtime, + dt=DT, + output_file=out_file, + verbose_progress=True, + ) diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 098132419..070cdb788 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -1,6 +1,11 @@ from dataclasses import dataclass +from datetime import timedelta +from pathlib import Path from typing import ClassVar +import numpy as np +from parcels import FieldSet, JITParticle, ParticleSet, Variable + from virtualship.models import Spacetime, instruments ## TODO: __init__.py will also need updating! @@ -17,9 +22,38 @@ class CTD: max_depth: float -# --------------- -# TODO: KERNELS -# --------------- +_CTDParticle = JITParticle.add_variables( + [ + Variable("salinity", dtype=np.float32, initial=np.nan), + Variable("temperature", dtype=np.float32, initial=np.nan), + Variable("raising", dtype=np.int8, initial=0.0), # bool. 0 is False, 1 is True. + Variable("max_depth", dtype=np.float32), + Variable("min_depth", dtype=np.float32), + Variable("winch_speed", dtype=np.float32), + ] +) + + +def _sample_temperature(particle, fieldset, time): + particle.temperature = fieldset.T[time, particle.depth, particle.lat, particle.lon] + + +def _sample_salinity(particle, fieldset, time): + particle.salinity = fieldset.S[time, particle.depth, particle.lat, particle.lon] + + +def _ctd_cast(particle, fieldset, time): + # lowering + if particle.raising == 0: + particle_ddepth = -particle.winch_speed * particle.dt + if particle.depth + particle_ddepth < particle.max_depth: + particle.raising = 1 + particle_ddepth = -particle_ddepth + # raising + else: + particle_ddepth = particle.winch_speed * particle.dt + if particle.depth + particle_ddepth > particle.min_depth: + particle.delete() class CTDInputDataset(instruments.InputDataset): @@ -59,18 +93,102 @@ def get_datasets_dict(self) -> dict: } -class CTDInstrument(instruments.Instrument): - """CTD instrument class.""" +# class CTDInstrument(instruments.Instrument): +# """CTD instrument class.""" + +# def __init__( +# self, +# config, +# input_dataset, +# kernels, +# ): +# """Initialise with instrument's name.""" +# super().__init__(CTD.name, config, input_dataset, kernels) + +# def simulate(self): +# """Simulate measurements.""" +# ... + + +def simulate_ctd( + fieldset: FieldSet, + out_path: str | Path, + ctds: list[CTD], + outputdt: timedelta, +) -> None: + """ + Use Parcels to simulate a set of CTDs in a fieldset. + + :param fieldset: The fieldset to simulate the CTDs in. + :param out_path: The path to write the results to. + :param ctds: A list of CTDs to simulate. + :param outputdt: Interval which dictates the update frequency of file output during simulation + :raises ValueError: Whenever provided CTDs, fieldset, are not compatible with this function. + """ + WINCH_SPEED = 1.0 # sink and rise speed in m/s + DT = 10.0 # dt of CTD simulation integrator + + if len(ctds) == 0: + print( + "No CTDs provided. Parcels currently crashes when providing an empty particle set, so no CTD simulation will be done and no files will be created." + ) + # TODO when Parcels supports it this check can be removed. + return + + fieldset_starttime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[0]) + fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) - def __init__( - self, - config, - input_dataset, - kernels, + # deploy time for all ctds should be later than fieldset start time + if not all( + [np.datetime64(ctd.spacetime.time) >= fieldset_starttime for ctd in ctds] ): - """Initialise with instrument's name.""" - super().__init__(CTD.name, config, input_dataset, kernels) + raise ValueError("CTD deployed before fieldset starts.") + + # depth the ctd will go to. shallowest between ctd max depth and bathymetry. + max_depths = [ + max( + ctd.max_depth, + fieldset.bathymetry.eval( + z=0, y=ctd.spacetime.location.lat, x=ctd.spacetime.location.lon, time=0 + ), + ) + for ctd in ctds + ] + + # CTD depth can not be too shallow, because kernel would break. + # This shallow is not useful anyway, no need to support. + if not all([max_depth <= -DT * WINCH_SPEED for max_depth in max_depths]): + raise ValueError( + f"CTD max_depth or bathymetry shallower than maximum {-DT * WINCH_SPEED}" + ) - def simulate(self): - """Simulate measurements.""" - ... + # define parcel particles + ctd_particleset = ParticleSet( + fieldset=fieldset, + pclass=_CTDParticle, + lon=[ctd.spacetime.location.lon for ctd in ctds], + lat=[ctd.spacetime.location.lat for ctd in ctds], + depth=[ctd.min_depth for ctd in ctds], + time=[ctd.spacetime.time for ctd in ctds], + max_depth=max_depths, + min_depth=[ctd.min_depth for ctd in ctds], + winch_speed=[WINCH_SPEED for _ in ctds], + ) + + # define output file for the simulation + out_file = ctd_particleset.ParticleFile(name=out_path, outputdt=outputdt) + + # execute simulation + ctd_particleset.execute( + [_sample_salinity, _sample_temperature, _ctd_cast], + endtime=fieldset_endtime, + dt=DT, + verbose_progress=False, + output_file=out_file, + ) + + # there should be no particles left, as they delete themselves when they resurface + if len(ctd_particleset.particledata) != 0: + raise ValueError( + "Simulation ended before CTD resurfaced. This most likely means the field time dimension did not match the simulation time span." + ) diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 77be7a060..1025a5c8d 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -1,6 +1,11 @@ from dataclasses import dataclass +from datetime import timedelta +from pathlib import Path from typing import ClassVar +import numpy as np +from parcels import FieldSet, JITParticle, ParticleSet, Variable + from virtualship.models import Spacetime, instruments @@ -14,9 +19,68 @@ class CTD_BGC: max_depth: float -# --------------- -# TODO: KERNELS -# --------------- +_CTD_BGCParticle = JITParticle.add_variables( + [ + Variable("o2", dtype=np.float32, initial=np.nan), + Variable("chl", dtype=np.float32, initial=np.nan), + Variable("no3", dtype=np.float32, initial=np.nan), + Variable("po4", dtype=np.float32, initial=np.nan), + Variable("ph", dtype=np.float32, initial=np.nan), + Variable("phyc", dtype=np.float32, initial=np.nan), + Variable("zooc", dtype=np.float32, initial=np.nan), + Variable("nppv", dtype=np.float32, initial=np.nan), + Variable("raising", dtype=np.int8, initial=0.0), # bool. 0 is False, 1 is True. + Variable("max_depth", dtype=np.float32), + Variable("min_depth", dtype=np.float32), + Variable("winch_speed", dtype=np.float32), + ] +) + + +def _sample_o2(particle, fieldset, time): + particle.o2 = fieldset.o2[time, particle.depth, particle.lat, particle.lon] + + +def _sample_chlorophyll(particle, fieldset, time): + particle.chl = fieldset.chl[time, particle.depth, particle.lat, particle.lon] + + +def _sample_nitrate(particle, fieldset, time): + particle.no3 = fieldset.no3[time, particle.depth, particle.lat, particle.lon] + + +def _sample_phosphate(particle, fieldset, time): + particle.po4 = fieldset.po4[time, particle.depth, particle.lat, particle.lon] + + +def _sample_ph(particle, fieldset, time): + particle.ph = fieldset.ph[time, particle.depth, particle.lat, particle.lon] + + +def _sample_phytoplankton(particle, fieldset, time): + particle.phyc = fieldset.phyc[time, particle.depth, particle.lat, particle.lon] + + +def _sample_zooplankton(particle, fieldset, time): + particle.zooc = fieldset.zooc[time, particle.depth, particle.lat, particle.lon] + + +def _sample_primary_production(particle, fieldset, time): + particle.nppv = fieldset.nppv[time, particle.depth, particle.lat, particle.lon] + + +def _ctd_bgc_cast(particle, fieldset, time): + # lowering + if particle.raising == 0: + particle_ddepth = -particle.winch_speed * particle.dt + if particle.depth + particle_ddepth < particle.max_depth: + particle.raising = 1 + particle_ddepth = -particle_ddepth + # raising + else: + particle_ddepth = particle.winch_speed * particle.dt + if particle.depth + particle_ddepth > particle.min_depth: + particle.delete() class CTD_BGCInputDataset(instruments.InputDataset): @@ -40,7 +104,7 @@ def __init__(self, data_dir, credentials, space_time_region): space_time_region, ) - def datasets_dir(self) -> dict: + def get_datasets_dict(self) -> dict: """Variable specific args for instrument.""" return { "o2data": { @@ -86,18 +150,118 @@ def datasets_dir(self) -> dict: } -class CTD_BGCInstrument(instruments.Instrument): - """CTD_BGC instrument class.""" +# class CTD_BGCInstrument(instruments.Instrument): +# """CTD_BGC instrument class.""" + +# def __init__( +# self, +# config, +# input_dataset, +# kernels, +# ): +# """Initialise with instrument's name.""" +# super().__init__(CTD_BGC.name, config, input_dataset, kernels) + +# def simulate(self): +# """Simulate measurements.""" +# ... + + +def simulate_ctd_bgc( + fieldset: FieldSet, + out_path: str | Path, + ctd_bgcs: list[CTD_BGC], + outputdt: timedelta, +) -> None: + """ + Use Parcels to simulate a set of BGC CTDs in a fieldset. + + :param fieldset: The fieldset to simulate the BGC CTDs in. + :param out_path: The path to write the results to. + :param ctds: A list of BGC CTDs to simulate. + :param outputdt: Interval which dictates the update frequency of file output during simulation + :raises ValueError: Whenever provided BGC CTDs, fieldset, are not compatible with this function. + """ + WINCH_SPEED = 1.0 # sink and rise speed in m/s + DT = 10.0 # dt of CTD simulation integrator + + if len(ctd_bgcs) == 0: + print( + "No BGC CTDs provided. Parcels currently crashes when providing an empty particle set, so no BGC CTD simulation will be done and no files will be created." + ) + # TODO when Parcels supports it this check can be removed. + return + + fieldset_starttime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[0]) + fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) - def __init__( - self, - config, - input_dataset, - kernels, + # deploy time for all ctds should be later than fieldset start time + if not all( + [ + np.datetime64(ctd_bgc.spacetime.time) >= fieldset_starttime + for ctd_bgc in ctd_bgcs + ] ): - """Initialise with instrument's name.""" - super().__init__(CTD_BGC.name, config, input_dataset, kernels) + raise ValueError("BGC CTD deployed before fieldset starts.") + + # depth the bgc ctd will go to. shallowest between bgc ctd max depth and bathymetry. + max_depths = [ + max( + ctd_bgc.max_depth, + fieldset.bathymetry.eval( + z=0, + y=ctd_bgc.spacetime.location.lat, + x=ctd_bgc.spacetime.location.lon, + time=0, + ), + ) + for ctd_bgc in ctd_bgcs + ] + + # CTD depth can not be too shallow, because kernel would break. + # This shallow is not useful anyway, no need to support. + if not all([max_depth <= -DT * WINCH_SPEED for max_depth in max_depths]): + raise ValueError( + f"BGC CTD max_depth or bathymetry shallower than maximum {-DT * WINCH_SPEED}" + ) - def simulate(self): - """Simulate measurements.""" - ... + # define parcel particles + ctd_bgc_particleset = ParticleSet( + fieldset=fieldset, + pclass=_CTD_BGCParticle, + lon=[ctd_bgc.spacetime.location.lon for ctd_bgc in ctd_bgcs], + lat=[ctd_bgc.spacetime.location.lat for ctd_bgc in ctd_bgcs], + depth=[ctd_bgc.min_depth for ctd_bgc in ctd_bgcs], + time=[ctd_bgc.spacetime.time for ctd_bgc in ctd_bgcs], + max_depth=max_depths, + min_depth=[ctd_bgc.min_depth for ctd_bgc in ctd_bgcs], + winch_speed=[WINCH_SPEED for _ in ctd_bgcs], + ) + + # define output file for the simulation + out_file = ctd_bgc_particleset.ParticleFile(name=out_path, outputdt=outputdt) + + # execute simulation + ctd_bgc_particleset.execute( + [ + _sample_o2, + _sample_chlorophyll, + _sample_nitrate, + _sample_phosphate, + _sample_ph, + _sample_phytoplankton, + _sample_zooplankton, + _sample_primary_production, + _ctd_bgc_cast, + ], + endtime=fieldset_endtime, + dt=DT, + verbose_progress=False, + output_file=out_file, + ) + + # there should be no particles left, as they delete themselves when they resurface + if len(ctd_bgc_particleset.particledata) != 0: + raise ValueError( + "Simulation ended before BGC CTD resurfaced. This most likely means the field time dimension did not match the simulation time span." + ) diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index 70ac4b7bb..d1e24390d 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -1,7 +1,11 @@ from dataclasses import dataclass -from datetime import timedelta +from datetime import datetime, timedelta +from pathlib import Path from typing import ClassVar +import numpy as np +from parcels import AdvectionRK4, FieldSet, JITParticle, ParticleSet, Variable + from virtualship.models import Spacetime, instruments ## TODO: __init__.py will also need updating! @@ -18,9 +22,25 @@ class Drifter: lifetime: timedelta | None # if none, lifetime is infinite -# --------------- -# TODO: KERNELS -# --------------- +_DrifterParticle = JITParticle.add_variables( + [ + Variable("temperature", dtype=np.float32, initial=np.nan), + Variable("has_lifetime", dtype=np.int8), # bool + Variable("age", dtype=np.float32, initial=0.0), + Variable("lifetime", dtype=np.float32), + ] +) + + +def _sample_temperature(particle, fieldset, time): + particle.temperature = fieldset.T[time, particle.depth, particle.lat, particle.lon] + + +def _check_lifetime(particle, fieldset, time): + if particle.has_lifetime == 1: + particle.age += particle.dt + if particle.age >= particle.lifetime: + particle.delete() class DrifterInputDataset(instruments.InputDataset): @@ -62,18 +82,91 @@ def get_datasets_dict(self) -> dict: } -class DrifterInstrument(instruments.Instrument): - """Drifter instrument class.""" - - def __init__( - self, - config, - input_dataset, - kernels, +# class DrifterInstrument(instruments.Instrument): +# """Drifter instrument class.""" + +# def __init__( +# self, +# config, +# input_dataset, +# kernels, +# ): +# """Initialise with instrument's name.""" +# super().__init__(Drifter.name, config, input_dataset, kernels) + +# def simulate(self): +# """Simulate measurements.""" +# ... + + +def simulate_drifters( + fieldset: FieldSet, + out_path: str | Path, + drifters: list[Drifter], + outputdt: timedelta, + dt: timedelta, + endtime: datetime | None = None, +) -> None: + """ + Use Parcels to simulate a set of drifters in a fieldset. + + :param fieldset: The fieldset to simulate the Drifters in. + :param out_path: The path to write the results to. + :param drifters: A list of drifters to simulate. + :param outputdt: Interval which dictates the update frequency of file output during simulation. + :param dt: Dt for integration. + :param endtime: Stop at this time, or if None, continue until the end of the fieldset or until all drifters ended. If this is earlier than the last drifter ended or later than the end of the fieldset, a warning will be printed. + """ + if len(drifters) == 0: + print( + "No drifters provided. Parcels currently crashes when providing an empty particle set, so no drifter simulation will be done and no files will be created." + ) + # TODO when Parcels supports it this check can be removed. + return + + # define parcel particles + drifter_particleset = ParticleSet( + fieldset=fieldset, + pclass=_DrifterParticle, + lat=[drifter.spacetime.location.lat for drifter in drifters], + lon=[drifter.spacetime.location.lon for drifter in drifters], + depth=[drifter.depth for drifter in drifters], + time=[drifter.spacetime.time for drifter in drifters], + has_lifetime=[1 if drifter.lifetime is not None else 0 for drifter in drifters], + lifetime=[ + 0 if drifter.lifetime is None else drifter.lifetime.total_seconds() + for drifter in drifters + ], + ) + + # define output file for the simulation + out_file = drifter_particleset.ParticleFile( + name=out_path, outputdt=outputdt, chunks=[len(drifter_particleset), 100] + ) + + # get earliest between fieldset end time and provide end time + fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) + if endtime is None: + actual_endtime = fieldset_endtime + elif endtime > fieldset_endtime: + print("WARN: Requested end time later than fieldset end time.") + actual_endtime = fieldset_endtime + else: + actual_endtime = np.timedelta64(endtime) + + # execute simulation + drifter_particleset.execute( + [AdvectionRK4, _sample_temperature, _check_lifetime], + endtime=actual_endtime, + dt=dt, + output_file=out_file, + verbose_progress=True, + ) + + # if there are more particles left than the number of drifters with an indefinite endtime, warn the user + if len(drifter_particleset.particledata) > len( + [d for d in drifters if d.lifetime is None] ): - """Initialise with instrument's name.""" - super().__init__(Drifter.name, config, input_dataset, kernels) - - def simulate(self): - """Simulate measurements.""" - ... + print( + "WARN: Some drifters had a life time beyond the end time of the fieldset or the requested end time." + ) diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index f364b1560..571a9ccd3 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -1,7 +1,11 @@ from dataclasses import dataclass +from pathlib import Path from typing import ClassVar -from virtualship.models import instruments +import numpy as np +from parcels import FieldSet, ParticleSet, ScipyParticle, Variable + +from virtualship.models import Spacetime, instruments ## TODO: __init__.py will also need updating! # + therefore instructions for adding new instruments will also involve adding to __init__.py as well as the new instrument script + update InstrumentType in instruments.py @@ -14,9 +18,22 @@ class Underwater_ST: name: ClassVar[str] = "Underwater_ST" -# --------------- -# TODO: KERNELS -# --------------- +_ShipSTParticle = ScipyParticle.add_variables( + [ + Variable("S", dtype=np.float32, initial=np.nan), + Variable("T", dtype=np.float32, initial=np.nan), + ] +) + + +# define function sampling Salinity +def _sample_salinity(particle, fieldset, time): + particle.S = fieldset.S[time, particle.depth, particle.lat, particle.lon] + + +# define function sampling Temperature +def _sample_temperature(particle, fieldset, time): + particle.T = fieldset.T[time, particle.depth, particle.lat, particle.lon] class Underwater_STInputDataset(instruments.InputDataset): @@ -58,18 +75,67 @@ def get_datasets_dict(self) -> dict: } -class Underwater_STInstrument(instruments.Instrument): - """Underwater_ST instrument class.""" - - def __init__( - self, - config, - input_dataset, - kernels, - ): - """Initialise with instrument's name.""" - super().__init__(Underwater_ST.name, config, input_dataset, kernels) +# class Underwater_STInstrument(instruments.Instrument): +# """Underwater_ST instrument class.""" + +# def __init__( +# self, +# config, +# input_dataset, +# kernels, +# ): +# """Initialise with instrument's name.""" +# super().__init__(Underwater_ST.name, config, input_dataset, kernels) + +# def simulate(self): +# """Simulate measurements.""" +# ... + + +def simulate_ship_underwater_st( + fieldset: FieldSet, + out_path: str | Path, + depth: float, + sample_points: list[Spacetime], +) -> None: + """ + Use Parcels to simulate underway data, measuring salinity and temperature at the given depth along the ship track in a fieldset. + + :param fieldset: The fieldset to simulate the sampling in. + :param out_path: The path to write the results to. + :param depth: The depth at which to measure. 0 is water surface, negative is into the water. + :param sample_points: The places and times to sample at. + """ + sample_points.sort(key=lambda p: p.time) + + particleset = ParticleSet.from_list( + fieldset=fieldset, + pclass=_ShipSTParticle, + lon=0.0, # initial lat/lon are irrelevant and will be overruled later + lat=0.0, + depth=depth, + time=0, # same for time + ) + + # define output file for the simulation + # outputdt set to infinie as we want to just want to write at the end of every call to 'execute' + out_file = particleset.ParticleFile(name=out_path, outputdt=np.inf) + + # iterate over each point, manually set lat lon time, then + # execute the particle set for one step, performing one set of measurement + for point in sample_points: + particleset.lon_nextloop[:] = point.location.lon + particleset.lat_nextloop[:] = point.location.lat + particleset.time_nextloop[:] = fieldset.time_origin.reltime( + np.datetime64(point.time) + ) - def simulate(self): - """Simulate measurements.""" - ... + # perform one step using the particleset + # dt and runtime are set so exactly one step is made. + particleset.execute( + [_sample_salinity, _sample_temperature], + dt=1, + runtime=1, + verbose_progress=False, + output_file=out_file, + ) diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index dcec018b4..4d90f3f14 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -1,7 +1,11 @@ from dataclasses import dataclass from datetime import timedelta +from pathlib import Path from typing import ClassVar +import numpy as np +from parcels import FieldSet, JITParticle, ParticleSet, Variable + from virtualship.models import Spacetime, instruments ## TODO: __init__.py will also need updating! @@ -18,9 +22,37 @@ class XBT: lifetime: timedelta | None # if none, lifetime is infinite -# --------------- -# TODO: KERNELS -# --------------- +_XBTParticle = JITParticle.add_variables( + [ + Variable("temperature", dtype=np.float32, initial=np.nan), + Variable("max_depth", dtype=np.float32), + Variable("min_depth", dtype=np.float32), + Variable("fall_speed", dtype=np.float32), + Variable("deceleration_coefficient", dtype=np.float32), + ] +) + + +def _sample_temperature(particle, fieldset, time): + particle.temperature = fieldset.T[time, particle.depth, particle.lat, particle.lon] + + +def _xbt_cast(particle, fieldset, time): + particle_ddepth = -particle.fall_speed * particle.dt + + # update the fall speed from the quadractic fall-rate equation + # check https://doi.org/10.5194/os-7-231-2011 + particle.fall_speed = ( + particle.fall_speed - 2 * particle.deceleration_coefficient * particle.dt + ) + + # delete particle if depth is exactly max_depth + if particle.depth == particle.max_depth: + particle.delete() + + # set particle depth to max depth if it's too deep + if particle.depth + particle_ddepth < particle.max_depth: + particle_ddepth = particle.max_depth - particle.depth class XBTInputDataset(instruments.InputDataset): @@ -67,18 +99,105 @@ def get_datasets_dict(self) -> dict: } -class XBTInstrument(instruments.Instrument): - """XBT instrument class.""" +# class XBTInstrument(instruments.Instrument): +# """XBT instrument class.""" + +# def __init__( +# self, +# config, +# input_dataset, +# kernels, +# ): +# """Initialise with instrument's name.""" +# super().__init__(XBT.name, config, input_dataset, kernels) + +# def simulate(self): +# """Simulate measurements.""" +# ... + + +def simulate_xbt( + fieldset: FieldSet, + out_path: str | Path, + xbts: list[XBT], + outputdt: timedelta, +) -> None: + """ + Use Parcels to simulate a set of XBTs in a fieldset. + + :param fieldset: The fieldset to simulate the XBTs in. + :param out_path: The path to write the results to. + :param xbts: A list of XBTs to simulate. + :param outputdt: Interval which dictates the update frequency of file output during simulation + :raises ValueError: Whenever provided XBTs, fieldset, are not compatible with this function. + """ + DT = 10.0 # dt of XBT simulation integrator + + if len(xbts) == 0: + print( + "No XBTs provided. Parcels currently crashes when providing an empty particle set, so no XBT simulation will be done and no files will be created." + ) + # TODO when Parcels supports it this check can be removed. + return - def __init__( - self, - config, - input_dataset, - kernels, - ): - """Initialise with instrument's name.""" - super().__init__(XBT.name, config, input_dataset, kernels) + fieldset_starttime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[0]) + fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) - def simulate(self): - """Simulate measurements.""" - ... + # deploy time for all xbts should be later than fieldset start time + if not all( + [np.datetime64(xbt.spacetime.time) >= fieldset_starttime for xbt in xbts] + ): + raise ValueError("XBT deployed before fieldset starts.") + + # depth the xbt will go to. shallowest between xbt max depth and bathymetry. + max_depths = [ + max( + xbt.max_depth, + fieldset.bathymetry.eval( + z=0, y=xbt.spacetime.location.lat, x=xbt.spacetime.location.lon, time=0 + ), + ) + for xbt in xbts + ] + + # initial fall speeds + initial_fall_speeds = [xbt.fall_speed for xbt in xbts] + + # XBT depth can not be too shallow, because kernel would break. + # This shallow is not useful anyway, no need to support. + for max_depth, fall_speed in zip(max_depths, initial_fall_speeds, strict=False): + if not max_depth <= -DT * fall_speed: + raise ValueError( + f"XBT max_depth or bathymetry shallower than maximum {-DT * fall_speed}" + ) + + # define xbt particles + xbt_particleset = ParticleSet( + fieldset=fieldset, + pclass=_XBTParticle, + lon=[xbt.spacetime.location.lon for xbt in xbts], + lat=[xbt.spacetime.location.lat for xbt in xbts], + depth=[xbt.min_depth for xbt in xbts], + time=[xbt.spacetime.time for xbt in xbts], + max_depth=max_depths, + min_depth=[xbt.min_depth for xbt in xbts], + fall_speed=[xbt.fall_speed for xbt in xbts], + ) + + # define output file for the simulation + out_file = xbt_particleset.ParticleFile(name=out_path, outputdt=outputdt) + + # execute simulation + xbt_particleset.execute( + [_sample_temperature, _xbt_cast], + endtime=fieldset_endtime, + dt=DT, + verbose_progress=False, + output_file=out_file, + ) + + # there should be no particles left, as they delete themselves when they finish profiling + if len(xbt_particleset.particledata) != 0: + raise ValueError( + "Simulation ended before XBT finished profiling. This most likely means the field time dimension did not match the simulation time span." + ) From 8b5954d46b6075cd6b44b5bb4fb9570b4169ca7d Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 9 Oct 2025 13:57:59 +0200 Subject: [PATCH 11/97] improve/clarify comments and notes --- src/virtualship/cli/_fetch.py | 6 ++++-- src/virtualship/cli/_plan.py | 1 - src/virtualship/instruments/ctd.py | 3 --- src/virtualship/instruments/drifter.py | 3 --- src/virtualship/instruments/master.py | 2 +- .../instruments/ship_underwater_st.py | 3 --- src/virtualship/models/instruments.py | 16 ++++------------ 7 files changed, 9 insertions(+), 25 deletions(-) diff --git a/src/virtualship/cli/_fetch.py b/src/virtualship/cli/_fetch.py index c258f0446..2a9be43e4 100644 --- a/src/virtualship/cli/_fetch.py +++ b/src/virtualship/cli/_fetch.py @@ -63,7 +63,7 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None expedition.schedule.space_time_region ) - # TODO: this (below) probably needs updating! + # TODO: needs updating? existing_download = get_existing_download(data_dir, space_time_region_hash) if existing_download is not None: click.echo( @@ -106,9 +106,11 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None ) shutil.copyfile(path / EXPEDITION, download_folder / EXPEDITION) + # TODO: enhance CLI output for users? + # bathymetry # TODO: this logic means it is downloaded for all expeditions but is only needed for CTD, CTD_BGC and XBT... - # TODO: to discuss: fine to still download for all expeditions because small size and then less duplication + # TODO: to discuss: fine to still download for all expeditions because small size and then less duplication? # TODO: or add as var in each of InputDataset objects per instrument because will be overwritten to disk anyway and therefore not duplicate? copernicusmarine.subset( dataset_id="cmems_mod_glo_phy_my_0.083deg_static", diff --git a/src/virtualship/cli/_plan.py b/src/virtualship/cli/_plan.py index 87bfe3363..1aa3208bf 100644 --- a/src/virtualship/cli/_plan.py +++ b/src/virtualship/cli/_plan.py @@ -847,7 +847,6 @@ def compose(self) -> ComposeResult: yield Select( [ (str(year), year) - # TODO: change from hard coding? ...flexibility for different datasets... for year in range( 2022, datetime.datetime.now().year + 1, diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 070cdb788..b2f59ac5e 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -8,9 +8,6 @@ from virtualship.models import Spacetime, instruments -## TODO: __init__.py will also need updating! -# + therefore instructions for adding new instruments will also involve adding to __init__.py as well as the new instrument script + update InstrumentType in instruments.py - @dataclass class CTD: diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index d1e24390d..8b0a1352a 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -8,9 +8,6 @@ from virtualship.models import Spacetime, instruments -## TODO: __init__.py will also need updating! -# + therefore instructions for adding new instruments will also involve adding to __init__.py as well as the new instrument script + update InstrumentType in instruments.py - @dataclass class Drifter: diff --git a/src/virtualship/instruments/master.py b/src/virtualship/instruments/master.py index 58eb05915..00f738915 100644 --- a/src/virtualship/instruments/master.py +++ b/src/virtualship/instruments/master.py @@ -6,7 +6,7 @@ class InstrumentType(Enum): # TODO: temporary measure so as not to have to overhaul the InstrumentType class logic in one go #! And also to avoid breaking other parts of the codebase which rely on InstrumentType when for now just working on fetch - # TODO: ideally this can evaporate in a future PR... + # TODO: ideally this can evaporate in the future... CTD = "CTD" CTD_BGC = "CTD_BGC" diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index 571a9ccd3..5fd39f2ae 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -7,9 +7,6 @@ from virtualship.models import Spacetime, instruments -## TODO: __init__.py will also need updating! -# + therefore instructions for adding new instruments will also involve adding to __init__.py as well as the new instrument script + update InstrumentType in instruments.py - @dataclass class Underwater_ST: diff --git a/src/virtualship/models/instruments.py b/src/virtualship/models/instruments.py index 7a3562908..72d9237de 100644 --- a/src/virtualship/models/instruments.py +++ b/src/virtualship/models/instruments.py @@ -9,18 +9,10 @@ from virtualship.models.space_time_region import SpaceTimeRegion from virtualship.utils import ship_spinner -# TODO list START -# how much detail needs to be fed into InputDataset (i.e. how much it differs per instrument) -# may impact whether need a child class (e.g. CTDInputDataset) as well as just InputDataset -# or whether it could just be fed a `name` ... ? - -# ++ abc.abstractmethods could be useful for testing purposes...e.g. will fail if an instrumnet implementation doesn't adhere to the `Instrument` class standards - -# ++ discussion point with others, do we think it's okay to overhaul the data downloading so that each instrument has it's own files, rather than sharing data? -# ++ it's a cleaner way of making the whole repo more modular, i.e. have higher order logic for defining data downloads and housing all instrument logic in one place... -# ++ may even not matter so much considering working towards cloud integration... + we are not looking to optimise performance...? -# ++ OR, for now work on it in this way and then at the end make some clever changes to consolidate to minimum number of files dependent on instrument selections...? -# TODO list END +# TODO: +# Discussion: Should each instrument manage its own data files for modularity, +# or should we consolidate downloads to minimize file duplication across instruments? +# Consider starting with per-instrument files for simplicity, and refactor later if needed. class InputDataset(abc.ABC): From ff38f4f261a03e4018a35ce67020b70b3a27d259 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 9 Oct 2025 13:58:25 +0200 Subject: [PATCH 12/97] Refactor ArgoFloat and XBT classes to include depth parameters and remove outdated comments --- src/virtualship/instruments/argo_float.py | 11 ++++++----- src/virtualship/instruments/xbt.py | 10 +++++----- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index 4769eccfa..a9f296d5c 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -16,9 +16,6 @@ from virtualship.models import Spacetime, instruments -## TODO: __init__.py will also need updating! -# + therefore instructions for adding new instruments will also involve adding to __init__.py as well as the new instrument script + update InstrumentType in instruments.py - @dataclass class ArgoFloat: @@ -26,8 +23,12 @@ class ArgoFloat: name: ClassVar[str] = "ArgoFloat" spacetime: Spacetime - depth: float # depth at which it floats and samples - lifetime: timedelta | None # if none, lifetime is infinite + min_depth: float + max_depth: float + drift_depth: float + vertical_speed: float + cycle_days: float + drift_days: float _ArgoParticle = JITParticle.add_variables( diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index 4d90f3f14..ca831afc0 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -8,9 +8,6 @@ from virtualship.models import Spacetime, instruments -## TODO: __init__.py will also need updating! -# + therefore instructions for adding new instruments will also involve adding to __init__.py as well as the new instrument script + update InstrumentType in instruments.py - @dataclass class XBT: @@ -18,8 +15,10 @@ class XBT: name: ClassVar[str] = "XBT" spacetime: Spacetime - depth: float # depth at which it floats and samples - lifetime: timedelta | None # if none, lifetime is infinite + min_depth: float + max_depth: float + fall_speed: float + deceleration_coefficient: float _XBTParticle = JITParticle.add_variables( @@ -165,6 +164,7 @@ def simulate_xbt( # XBT depth can not be too shallow, because kernel would break. # This shallow is not useful anyway, no need to support. + # TODO: should this be more informative? Is "maximum" right? Should tell user can't use XBT here? for max_depth, fall_speed in zip(max_depths, initial_fall_speeds, strict=False): if not max_depth <= -DT * fall_speed: raise ValueError( From c870d1cbcf5c218cb9d8637acb45bca045c0435a Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 14 Oct 2025 11:59:25 +0200 Subject: [PATCH 13/97] avoid circular import issues --- src/virtualship/instruments/__init__.py | 2 ++ src/virtualship/instruments/adcp.py | 5 +++-- src/virtualship/instruments/argo_float.py | 5 +++-- src/virtualship/instruments/ctd.py | 5 +++-- src/virtualship/instruments/ctd_bgc.py | 5 +++-- src/virtualship/instruments/drifter.py | 5 +++-- src/virtualship/instruments/ship_underwater_st.py | 5 +++-- src/virtualship/instruments/xbt.py | 5 +++-- 8 files changed, 23 insertions(+), 14 deletions(-) diff --git a/src/virtualship/instruments/__init__.py b/src/virtualship/instruments/__init__.py index 6a6ffbca3..a5da8dacd 100644 --- a/src/virtualship/instruments/__init__.py +++ b/src/virtualship/instruments/__init__.py @@ -1,5 +1,7 @@ """Measurement instrument that can be used with Parcels.""" +from virtualship.models.spacetime import Spacetime # noqa: F401 + from . import adcp, argo_float, ctd, ctd_bgc, drifter, ship_underwater_st, xbt __all__ = [ diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index 052004fba..ccf487389 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -5,7 +5,8 @@ import numpy as np from parcels import FieldSet, ParticleSet, ScipyParticle, Variable -from virtualship.models import Spacetime, instruments +from virtualship.models.instruments import InputDataset +from virtualship.models.spacetime import Spacetime ## TODO: __init__.py will also need updating! # + therefore instructions for adding new instruments will also involve adding to __init__.py as well as the new instrument script + update InstrumentType in instruments.py @@ -34,7 +35,7 @@ def _sample_velocity(particle, fieldset, time): ) -class ADCPInputDataset(instruments.InputDataset): +class ADCPInputDataset(InputDataset): """Input dataset for ADCP instrument.""" DOWNLOAD_BUFFERS: ClassVar[dict] = { diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index a9f296d5c..b9949fb16 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -14,7 +14,8 @@ Variable, ) -from virtualship.models import Spacetime, instruments +from virtualship.models.instruments import InputDataset +from virtualship.models.spacetime import Spacetime @dataclass @@ -115,7 +116,7 @@ def _check_error(particle, fieldset, time): particle.delete() -class ArgoFloatInputDataset(instruments.InputDataset): +class ArgoFloatInputDataset(InputDataset): """Input dataset for ArgoFloat instrument.""" DOWNLOAD_BUFFERS: ClassVar[dict] = { diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index b2f59ac5e..15a462349 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -6,7 +6,8 @@ import numpy as np from parcels import FieldSet, JITParticle, ParticleSet, Variable -from virtualship.models import Spacetime, instruments +from virtualship.models.instruments import InputDataset +from virtualship.models.spacetime import Spacetime @dataclass @@ -53,7 +54,7 @@ def _ctd_cast(particle, fieldset, time): particle.delete() -class CTDInputDataset(instruments.InputDataset): +class CTDInputDataset(InputDataset): """Input dataset for CTD instrument.""" DOWNLOAD_BUFFERS: ClassVar[dict] = { diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 1025a5c8d..fb9ea241d 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -6,7 +6,8 @@ import numpy as np from parcels import FieldSet, JITParticle, ParticleSet, Variable -from virtualship.models import Spacetime, instruments +from virtualship.models.instruments import InputDataset +from virtualship.models.spacetime import Spacetime @dataclass @@ -83,7 +84,7 @@ def _ctd_bgc_cast(particle, fieldset, time): particle.delete() -class CTD_BGCInputDataset(instruments.InputDataset): +class CTD_BGCInputDataset(InputDataset): """Input dataset object for CTD_BGC instrument.""" DOWNLOAD_BUFFERS: ClassVar[dict] = { diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index 8b0a1352a..60ba5558a 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -6,7 +6,8 @@ import numpy as np from parcels import AdvectionRK4, FieldSet, JITParticle, ParticleSet, Variable -from virtualship.models import Spacetime, instruments +from virtualship.models.instruments import InputDataset +from virtualship.models.spacetime import Spacetime @dataclass @@ -40,7 +41,7 @@ def _check_lifetime(particle, fieldset, time): particle.delete() -class DrifterInputDataset(instruments.InputDataset): +class DrifterInputDataset(InputDataset): """Input dataset for Drifter instrument.""" DOWNLOAD_BUFFERS: ClassVar[dict] = { diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index 5fd39f2ae..332c0b2c3 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -5,7 +5,8 @@ import numpy as np from parcels import FieldSet, ParticleSet, ScipyParticle, Variable -from virtualship.models import Spacetime, instruments +from virtualship.models.instruments import InputDataset +from virtualship.models.spacetime import Spacetime @dataclass @@ -33,7 +34,7 @@ def _sample_temperature(particle, fieldset, time): particle.T = fieldset.T[time, particle.depth, particle.lat, particle.lon] -class Underwater_STInputDataset(instruments.InputDataset): +class Underwater_STInputDataset(InputDataset): """Input dataset for Underwater_ST instrument.""" DOWNLOAD_BUFFERS: ClassVar[dict] = { diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index ca831afc0..81cba7b90 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -6,7 +6,8 @@ import numpy as np from parcels import FieldSet, JITParticle, ParticleSet, Variable -from virtualship.models import Spacetime, instruments +from virtualship.models.instruments import InputDataset +from virtualship.models.spacetime import Spacetime @dataclass @@ -54,7 +55,7 @@ def _xbt_cast(particle, fieldset, time): particle_ddepth = particle.max_depth - particle.depth -class XBTInputDataset(instruments.InputDataset): +class XBTInputDataset(InputDataset): """Input dataset for XBT instrument.""" DOWNLOAD_BUFFERS: ClassVar[dict] = { From 49b3beeb44f8938e16398828f8a1448384002d17 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 14 Oct 2025 12:01:27 +0200 Subject: [PATCH 14/97] make tests for InputDataset base class --- tests/models/test_instruments.py | 97 ++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 tests/models/test_instruments.py diff --git a/tests/models/test_instruments.py b/tests/models/test_instruments.py new file mode 100644 index 000000000..2ad73cdfc --- /dev/null +++ b/tests/models/test_instruments.py @@ -0,0 +1,97 @@ +import datetime +from unittest.mock import patch + +import pytest + +from virtualship.models.instruments import InputDataset +from virtualship.models.space_time_region import ( + SpaceTimeRegion, + SpatialRange, + TimeRange, +) + + +class DummyInputDataset(InputDataset): + """A minimal InputDataset subclass for testing purposes.""" + + def get_datasets_dict(self): + """Return a dummy datasets dict for testing.""" + return { + "dummy": { + "dataset_id": "test_id", + "variables": ["var1"], + "output_filename": "dummy.nc", + } + } + + +@pytest.fixture +def dummy_space_time_region(): + spatial_range = SpatialRange( + minimum_longitude=0, + maximum_longitude=1, + minimum_latitude=0, + maximum_latitude=1, + minimum_depth=0, + maximum_depth=10, + ) + base_time = datetime.datetime.strptime("1950-01-01", "%Y-%m-%d") + time_range = TimeRange( + start_time=base_time, + end_time=base_time + datetime.timedelta(hours=1), + ) + return SpaceTimeRegion( + spatial_range=spatial_range, + time_range=time_range, + ) + + +def test_inputdataset_abstract_instantiation(): + # instantiation should not be allowed + with pytest.raises(TypeError): + InputDataset( + name="test", + latlon_buffer=0, + datetime_buffer=0, + min_depth=0, + max_depth=10, + data_dir=".", + credentials={"username": "u", "password": "p"}, + space_time_region=None, + ) + + +def test_dummyinputdataset_initialization(dummy_space_time_region): + ds = DummyInputDataset( + name="test", + latlon_buffer=0.5, + datetime_buffer=1, + min_depth=0, + max_depth=10, + data_dir=".", + credentials={"username": "u", "password": "p"}, + space_time_region=dummy_space_time_region, + ) + assert ds.name == "test" + assert ds.latlon_buffer == 0.5 + assert ds.datetime_buffer == 1 + assert ds.min_depth == 0 + assert ds.max_depth == 10 + assert ds.data_dir == "." + assert ds.credentials["username"] == "u" + + +@patch("virtualship.models.instruments.copernicusmarine.subset") +def test_download_data_calls_subset(mock_subset, dummy_space_time_region): + ds = DummyInputDataset( + name="test", + latlon_buffer=0.5, + datetime_buffer=1, + min_depth=0, + max_depth=10, + data_dir=".", + credentials={"username": "u", "password": "p"}, + space_time_region=dummy_space_time_region, + ) + ds.download_data() + assert mock_subset.called From c4e31963d8d23f71c6e051228ff0b70cff3f8c0d Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 22 Oct 2025 10:40:39 +0200 Subject: [PATCH 15/97] refactor instrument handling in _fetch.py and update expedition model to include get_instruments method --- src/virtualship/cli/_fetch.py | 149 ++---------------------- src/virtualship/instruments/__init__.py | 16 +-- src/virtualship/instruments/master.py | 4 +- src/virtualship/models/__init__.py | 2 - src/virtualship/models/expedition.py | 39 +++---- 5 files changed, 30 insertions(+), 180 deletions(-) diff --git a/src/virtualship/cli/_fetch.py b/src/virtualship/cli/_fetch.py index 2a9be43e4..c460db379 100644 --- a/src/virtualship/cli/_fetch.py +++ b/src/virtualship/cli/_fetch.py @@ -11,6 +11,7 @@ from pydantic import BaseModel from virtualship.errors import IncompleteDownloadError +from virtualship.instruments.master import get_instruments_registry from virtualship.utils import ( _dump_yaml, _generic_load_yaml, @@ -24,8 +25,6 @@ import virtualship.cli._creds as creds from virtualship.utils import EXPEDITION -from virtualship.instruments.master import INSTRUMENTS -from virtualship.instruments.master import InstrumentType, get_instruments_registry DOWNLOAD_METADATA = "download_metadata.yaml" @@ -63,7 +62,6 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None expedition.schedule.space_time_region ) - # TODO: needs updating? existing_download = get_existing_download(data_dir, space_time_region_hash) if existing_download is not None: click.echo( @@ -77,26 +75,9 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None username, password, creds_path ) - # Extract space_time_region details from the schedule - spatial_range = expedition.schedule.space_time_region.spatial_range - time_range = expedition.schedule.space_time_region.time_range - start_datetime = time_range.start_time - end_datetime = time_range.end_time - instruments_in_schedule = expedition.schedule.get_instruments() - - # TEMPORARY measure to get underway instruments in `instruments_in_schedule` - # TODO: should evaporate when schedule and ship_config.yaml files are consolidated in a separate PR... - if ship_config.adcp_config is not None: - instruments_in_schedule.add(InstrumentType.ADCP) - if ship_config.ship_underwater_st_config is not None: - instruments_in_schedule.add(InstrumentType.UNDERWATER_ST) - - # TEMPORARY measure to get underway instruments in `instruments_in_schedule` - # TODO: should evaporate when schedule and ship_config.yaml files are consolidated in a separate PR... - if ship_config.adcp_config is not None: - instruments_in_schedule.add(InstrumentType.ADCP) - if ship_config.ship_underwater_st_config is not None: - instruments_in_schedule.add(InstrumentType.UNDERWATER_ST) + # Extract instruments and space_time_region details from expedition + instruments_in_expedition = expedition.get_instruments() + space_time_region = expedition.schedule.space_time_region # Create download folder and set download metadata download_folder = data_dir / hash_to_filename(space_time_region_hash) @@ -108,10 +89,7 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None # TODO: enhance CLI output for users? - # bathymetry - # TODO: this logic means it is downloaded for all expeditions but is only needed for CTD, CTD_BGC and XBT... - # TODO: to discuss: fine to still download for all expeditions because small size and then less duplication? - # TODO: or add as var in each of InputDataset objects per instrument because will be overwritten to disk anyway and therefore not duplicate? + # bathymetry (for all expeditions) copernicusmarine.subset( dataset_id="cmems_mod_glo_phy_my_0.083deg_static", variables=["deptho"], @@ -131,9 +109,9 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None coordinates_selection_method="outside", ) - # keep only instruments in INTSTRUMENTS which are in schedule + # access instrument classes but keep only instruments which are in schedule filter_instruments = { - k: v for k, v in INSTRUMENTS.items() if k in instruments_in_schedule + k: v for k, v in INSTRUMENTS.items() if k in instruments_in_expedition } # iterate across instruments and download data based on space_time_region @@ -142,116 +120,11 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None input_dataset = instrument["input_class"]( data_dir=download_folder, credentials=credentials, - space_time_region=space_time_region, + space_time_region=expedition.space_time_region, ) - #! - #### TODO - # ++ new logic here where iterates (?) through available instruments and determines whether download is required: - # ++ by conditions of: - # 1) whether it's in the schedule (and from this be able to call the right classes from the instruments directory?) and - #! 2) is there a clever way of not unnecessarily duplicating data downloads if instruments use the same?! - # (try with a version first where does them all in tow and then try and optimise...?) - - #! - ## TODO: move to generic bathymetry download which is done for all expeditions - - if ( - ( - {"XBT", "CTD", "CDT_BGC", "SHIP_UNDERWATER_ST"} - & set(instrument.name for instrument in instruments_in_schedule) - ) - or expedition.instruments_config.ship_underwater_st_config is not None - or expedition.instruments_config.adcp_config is not None - ): - print("Ship data will be downloaded. Please wait...") - - # Define all ship datasets to download, including bathymetry - download_dict = { - "Bathymetry": { - "dataset_id": "cmems_mod_glo_phy_my_0.083deg_static", - "variables": ["deptho"], - "output_filename": "bathymetry.nc", - }, - "UVdata": { - "dataset_id": "cmems_mod_glo_phy-cur_anfc_0.083deg_PT6H-i", - "variables": ["uo", "vo"], - "output_filename": "ship_uv.nc", - }, - "Sdata": { - "dataset_id": "cmems_mod_glo_phy-so_anfc_0.083deg_PT6H-i", - "variables": ["so"], - "output_filename": "ship_s.nc", - }, - "Tdata": { - "dataset_id": "cmems_mod_glo_phy-thetao_anfc_0.083deg_PT6H-i", - "variables": ["thetao"], - "output_filename": "ship_t.nc", - }, - } - - # Iterate over all datasets and download each based on space_time_region - try: - for dataset in download_dict.values(): - copernicusmarine.subset( - dataset_id=dataset["dataset_id"], - variables=dataset["variables"], - minimum_longitude=spatial_range.minimum_longitude, - maximum_longitude=spatial_range.maximum_longitude, - minimum_latitude=spatial_range.minimum_latitude, - maximum_latitude=spatial_range.maximum_latitude, - start_datetime=start_datetime, - end_datetime=end_datetime, - minimum_depth=abs(spatial_range.minimum_depth), - maximum_depth=abs(spatial_range.maximum_depth), - output_filename=dataset["output_filename"], - output_directory=download_folder, - username=username, - password=password, - overwrite=True, - coordinates_selection_method="outside", - ) - except InvalidUsernameOrPassword as e: - shutil.rmtree(download_folder) - raise e - click.echo("Ship data download based on space-time region completed.") - - if InstrumentType.DRIFTER in instruments_in_schedule: - print("Drifter data will be downloaded. Please wait...") - drifter_download_dict = { - "UVdata": { - "dataset_id": "cmems_mod_glo_phy-cur_anfc_0.083deg_PT6H-i", - "variables": ["uo", "vo"], - "output_filename": "drifter_uv.nc", - }, - "Tdata": { - "dataset_id": "cmems_mod_glo_phy-thetao_anfc_0.083deg_PT6H-i", - "variables": ["thetao"], - "output_filename": "drifter_t.nc", - }, - } - - # Iterate over all datasets and download each based on space_time_region - try: - for dataset in drifter_download_dict.values(): - copernicusmarine.subset( - dataset_id=dataset["dataset_id"], - variables=dataset["variables"], - minimum_longitude=spatial_range.minimum_longitude - 3.0, - maximum_longitude=spatial_range.maximum_longitude + 3.0, - minimum_latitude=spatial_range.minimum_latitude - 3.0, - maximum_latitude=spatial_range.maximum_latitude + 3.0, - start_datetime=start_datetime, - end_datetime=end_datetime + timedelta(days=21), - minimum_depth=abs(1), - maximum_depth=abs(1), - output_filename=dataset["output_filename"], - output_directory=download_folder, - username=username, - password=password, - overwrite=True, - coordinates_selection_method="outside", - ) + input_dataset.download_data() + except InvalidUsernameOrPassword as e: shutil.rmtree(download_folder) raise e @@ -331,11 +204,9 @@ def get_existing_download(data_dir: Path, space_time_region_hash: str) -> Path | hash = filename_to_hash(download_path.name) except ValueError: continue - if hash == space_time_region_hash: assert_complete_download(download_path) return download_path - return None diff --git a/src/virtualship/instruments/__init__.py b/src/virtualship/instruments/__init__.py index a5da8dacd..5324da2cd 100644 --- a/src/virtualship/instruments/__init__.py +++ b/src/virtualship/instruments/__init__.py @@ -1,15 +1 @@ -"""Measurement instrument that can be used with Parcels.""" - -from virtualship.models.spacetime import Spacetime # noqa: F401 - -from . import adcp, argo_float, ctd, ctd_bgc, drifter, ship_underwater_st, xbt - -__all__ = [ - "adcp", - "argo_float", - "ctd", - "ctd_bgc", - "drifter", - "ship_underwater_st", - "xbt", -] +"""Instruments in VirtualShip.""" diff --git a/src/virtualship/instruments/master.py b/src/virtualship/instruments/master.py index 00f738915..cb31468f1 100644 --- a/src/virtualship/instruments/master.py +++ b/src/virtualship/instruments/master.py @@ -4,9 +4,7 @@ class InstrumentType(Enum): """Types of the instruments.""" - # TODO: temporary measure so as not to have to overhaul the InstrumentType class logic in one go - #! And also to avoid breaking other parts of the codebase which rely on InstrumentType when for now just working on fetch - # TODO: ideally this can evaporate in the future... + # TODO: scope for this to evaporate in the future...? CTD = "CTD" CTD_BGC = "CTD_BGC" diff --git a/src/virtualship/models/__init__.py b/src/virtualship/models/__init__.py index a2f1546cb..5eaabb852 100644 --- a/src/virtualship/models/__init__.py +++ b/src/virtualship/models/__init__.py @@ -8,7 +8,6 @@ DrifterConfig, Expedition, InstrumentsConfig, - InstrumentType, Schedule, ShipConfig, ShipUnderwaterSTConfig, @@ -30,7 +29,6 @@ "Schedule", "ShipConfig", "Waypoint", - "InstrumentType", "ArgoFloatConfig", "ADCPConfig", "CTDConfig", diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index 2e073b842..ef860625d 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -2,7 +2,6 @@ import itertools from datetime import datetime, timedelta -from enum import Enum from typing import TYPE_CHECKING import pydantic @@ -10,6 +9,7 @@ import yaml from virtualship.errors import ConfigError, ScheduleError +from virtualship.instruments.master import InstrumentType from virtualship.utils import _validate_numeric_mins_to_timedelta from .location import Location @@ -45,6 +45,23 @@ def from_yaml(cls, file_path: str) -> Expedition: data = yaml.safe_load(file) return Expedition(**data) + def get_instruments(self) -> set[InstrumentType]: + """Return a set of unique InstrumentType enums used in the expedition.""" + instruments_in_expedition = [] + # from waypoints + for waypoint in self.schedule.waypoints: + if waypoint.instrument: + for instrument in waypoint.instrument: + if instrument: + instruments_in_expedition.append(instrument) + # check for underway instruments and add if present in expeditions + if self.instruments_config.adcp_config is not None: + instruments_in_expedition.append(InstrumentType.ADCP) + if self.instruments_config.ship_underwater_st_config is not None: + instruments_in_expedition.append(InstrumentType.UNDERWATER_ST) + + return set(instruments_in_expedition) + class ShipConfig(pydantic.BaseModel): """Configuration of the ship.""" @@ -64,16 +81,6 @@ class Schedule(pydantic.BaseModel): model_config = pydantic.ConfigDict(extra="forbid") - def get_instruments(self) -> set[InstrumentType]: - """Return a set of unique InstrumentType enums used in the schedule.""" - instruments_in_schedule = [] - for waypoint in self.waypoints: - if waypoint.instrument: - for instrument in waypoint.instrument: - if instrument: - instruments_in_schedule.append(instrument) - return set(instruments_in_schedule) - def verify( self, ship_speed: float, @@ -213,16 +220,6 @@ def serialize_instrument(self, instrument): return instrument.value if instrument else None -class InstrumentType(Enum): - """Types of the instruments.""" - - CTD = "CTD" - CTD_BGC = "CTD_BGC" - DRIFTER = "DRIFTER" - ARGO_FLOAT = "ARGO_FLOAT" - XBT = "XBT" - - class ArgoFloatConfig(pydantic.BaseModel): """Configuration for argos floats.""" From 2f40f7dff5755e5e173800e8ee81a95ab5640fdf Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 22 Oct 2025 15:21:02 +0200 Subject: [PATCH 16/97] refactor instrument error handling in Expedition model and remove Schedule and ShipConfig classes --- src/virtualship/models/expedition.py | 37 +-- src/virtualship/models/schedule.py | 236 -------------------- src/virtualship/models/ship_config.py | 310 -------------------------- 3 files changed, 24 insertions(+), 559 deletions(-) delete mode 100644 src/virtualship/models/schedule.py delete mode 100644 src/virtualship/models/ship_config.py diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index ef860625d..7e206e8f1 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -8,7 +8,7 @@ import pyproj import yaml -from virtualship.errors import ConfigError, ScheduleError +from virtualship.errors import InstrumentsConfigError, ScheduleError from virtualship.instruments.master import InstrumentType from virtualship.utils import _validate_numeric_mins_to_timedelta @@ -54,13 +54,18 @@ def get_instruments(self) -> set[InstrumentType]: for instrument in waypoint.instrument: if instrument: instruments_in_expedition.append(instrument) - # check for underway instruments and add if present in expeditions - if self.instruments_config.adcp_config is not None: - instruments_in_expedition.append(InstrumentType.ADCP) - if self.instruments_config.ship_underwater_st_config is not None: - instruments_in_expedition.append(InstrumentType.UNDERWATER_ST) - return set(instruments_in_expedition) + # check for underway instruments and add if present in expeditions + try: + if self.instruments_config.adcp_config is not None: + instruments_in_expedition.append(InstrumentType.ADCP) + if self.instruments_config.ship_underwater_st_config is not None: + instruments_in_expedition.append(InstrumentType.UNDERWATER_ST) + return set(instruments_in_expedition) + except Exception as e: + raise InstrumentsConfigError( + "Underway instrument config attribute(s) are missing from YAML. Must be Config object or None." + ) from e class ShipConfig(pydantic.BaseModel): @@ -348,6 +353,7 @@ class XBTConfig(pydantic.BaseModel): class InstrumentsConfig(pydantic.BaseModel): + # TODO: refactor potential for this? Move explicit instrument_config's away from models/ dir? """Configuration of instruments.""" argo_float_config: ArgoFloatConfig | None = None @@ -401,38 +407,43 @@ class InstrumentsConfig(pydantic.BaseModel): model_config = pydantic.ConfigDict(extra="forbid") - def verify(self, schedule: Schedule) -> None: + def verify(self, expedition: Expedition) -> None: """ Verify instrument configurations against the schedule. Removes instrument configs not present in the schedule and checks that all scheduled instruments are configured. Raises ConfigError if any scheduled instrument is missing a config. """ - instruments_in_schedule = schedule.get_instruments() + instruments_in_expedition = expedition.get_instruments() instrument_config_map = { InstrumentType.ARGO_FLOAT: "argo_float_config", InstrumentType.DRIFTER: "drifter_config", InstrumentType.XBT: "xbt_config", InstrumentType.CTD: "ctd_config", InstrumentType.CTD_BGC: "ctd_bgc_config", + InstrumentType.ADCP: "adcp_config", + InstrumentType.UNDERWATER_ST: "ship_underwater_st_config", } # Remove configs for unused instruments for inst_type, config_attr in instrument_config_map.items(): - if hasattr(self, config_attr) and inst_type not in instruments_in_schedule: + if ( + hasattr(self, config_attr) + and inst_type not in instruments_in_expedition + ): print( f"{inst_type.value} configuration provided but not in schedule. Removing config." ) setattr(self, config_attr, None) # Check all scheduled instruments are configured - for inst_type in instruments_in_schedule: + for inst_type in instruments_in_expedition: config_attr = instrument_config_map.get(inst_type) if ( not config_attr or not hasattr(self, config_attr) or getattr(self, config_attr) is None ): - raise ConfigError( - f"Schedule includes instrument '{inst_type.value}', but instruments_config does not provide configuration for it." + raise InstrumentsConfigError( + f"Expedition includes instrument '{inst_type.value}', but instruments_config does not provide configuration for it." ) diff --git a/src/virtualship/models/schedule.py b/src/virtualship/models/schedule.py deleted file mode 100644 index f3e5dabea..000000000 --- a/src/virtualship/models/schedule.py +++ /dev/null @@ -1,236 +0,0 @@ -"""Schedule class.""" - -from __future__ import annotations - -import itertools -from datetime import datetime, timedelta -from pathlib import Path -from typing import TYPE_CHECKING - -import pydantic -import pyproj -import yaml - -from virtualship.errors import ScheduleError -from virtualship.instruments.master import InstrumentType - -from .location import Location -from .space_time_region import SpaceTimeRegion - -if TYPE_CHECKING: - from parcels import FieldSet - - from virtualship.expedition.input_data import InputData - -projection: pyproj.Geod = pyproj.Geod(ellps="WGS84") - - -class Waypoint(pydantic.BaseModel): - """A Waypoint to sail to with an optional time and an optional instrument.""" - - location: Location - time: datetime | None = None - instrument: InstrumentType | list[InstrumentType] | None = None - - @pydantic.field_serializer("instrument") - def serialize_instrument(self, instrument): - """Ensure InstrumentType is serialized as a string (or list of strings).""" - if isinstance(instrument, list): - return [inst.value for inst in instrument] - return instrument.value if instrument else None - - -class Schedule(pydantic.BaseModel): - """Schedule of the virtual ship.""" - - waypoints: list[Waypoint] - space_time_region: SpaceTimeRegion | None = None - - model_config = pydantic.ConfigDict(extra="forbid") - - def to_yaml(self, file_path: str | Path) -> None: - """ - Write schedule to yaml file. - - :param file_path: Path to the file to write to. - """ - with open(file_path, "w") as file: - yaml.dump( - self.model_dump( - by_alias=True, - ), - file, - ) - - @classmethod - def from_yaml(cls, file_path: str | Path) -> Schedule: - """ - Load schedule from yaml file. - - :param file_path: Path to the file to load from. - :returns: The schedule. - """ - with open(file_path) as file: - data = yaml.safe_load(file) - return Schedule(**data) - - def get_instruments(self) -> set[InstrumentType]: - """ - Retrieve a set of unique instruments used in the schedule. - - This method iterates through all waypoints in the schedule and collects - the instruments associated with each waypoint. It returns a set of unique - instruments, either as objects or as names. - - :raises CheckpointError: If the past waypoints in the given schedule - have been changed compared to the checkpoint. - :return: set: A set of unique instruments used in the schedule. - - """ - instruments_in_schedule = [] - for waypoint in self.waypoints: - if waypoint.instrument: - for instrument in waypoint.instrument: - if instrument: - instruments_in_schedule.append(instrument) - return set(instruments_in_schedule) - - def verify( - self, - ship_speed: float, - input_data: InputData | None, - *, - check_space_time_region: bool = False, - ignore_missing_fieldsets: bool = False, - ) -> None: - """ - Verify the feasibility and correctness of the schedule's waypoints. - - This method checks various conditions to ensure the schedule is valid: - 1. At least one waypoint is provided. - 2. The first waypoint has a specified time. - 3. Waypoint times are in ascending order. - 4. All waypoints are in water (not on land). - 5. The ship can arrive on time at each waypoint given its speed. - - :param ship_speed: The ship's speed in knots. - :param input_data: An InputData object containing fieldsets used to check if waypoints are on water. - :param check_space_time_region: whether to check for missing space_time_region. - :param ignore_missing_fieldsets: whether to ignore warning for missing field sets. - :raises PlanningError: If any of the verification checks fail, indicating infeasible or incorrect waypoints. - :raises NotImplementedError: If an instrument in the schedule is not implemented. - :return: None. The method doesn't return a value but raises exceptions if verification fails. - """ - print("\nVerifying route... ") - - if check_space_time_region and self.space_time_region is None: - raise ScheduleError( - "space_time_region not found in schedule, please define it to fetch the data." - ) - - if len(self.waypoints) == 0: - raise ScheduleError("At least one waypoint must be provided.") - - # check first waypoint has a time - if self.waypoints[0].time is None: - raise ScheduleError("First waypoint must have a specified time.") - - # check waypoint times are in ascending order - timed_waypoints = [wp for wp in self.waypoints if wp.time is not None] - checks = [ - next.time >= cur.time for cur, next in itertools.pairwise(timed_waypoints) - ] - if not all(checks): - invalid_i = [i for i, c in enumerate(checks) if c] - raise ScheduleError( - f"Waypoint(s) {', '.join(f'#{i + 1}' for i in invalid_i)}: each waypoint should be timed after all previous waypoints", - ) - - # check if all waypoints are in water - # this is done by picking an arbitrary provided fieldset and checking if UV is not zero - - # get all available fieldsets - available_fieldsets = [] - if input_data is not None: - fieldsets = [ - input_data.adcp_fieldset, - input_data.argo_float_fieldset, - input_data.ctd_fieldset, - input_data.drifter_fieldset, - input_data.ship_underwater_st_fieldset, - ] - for fs in fieldsets: - if fs is not None: - available_fieldsets.append(fs) - - # check if there are any fieldsets, else it's an error - if len(available_fieldsets) == 0: - if not ignore_missing_fieldsets: - print( - "Cannot verify because no fieldsets have been loaded. This is probably " - "because you are not using any instruments in your schedule. This is not a problem, " - "but carefully check your waypoint locations manually." - ) - - else: - # pick any - fieldset = available_fieldsets[0] - # get waypoints with 0 UV - land_waypoints = [ - (wp_i, wp) - for wp_i, wp in enumerate(self.waypoints) - if _is_on_land_zero_uv(fieldset, wp) - ] - # raise an error if there are any - if len(land_waypoints) > 0: - raise ScheduleError( - f"The following waypoints are on land: {['#' + str(wp_i) + ' ' + str(wp) for (wp_i, wp) in land_waypoints]}" - ) - - # check that ship will arrive on time at each waypoint (in case no unexpected event happen) - time = self.waypoints[0].time - for wp_i, (wp, wp_next) in enumerate( - zip(self.waypoints, self.waypoints[1:], strict=False) - ): - if wp.instrument is InstrumentType.CTD: - time += timedelta(minutes=20) - - geodinv: tuple[float, float, float] = projection.inv( - wp.location.lon, - wp.location.lat, - wp_next.location.lon, - wp_next.location.lat, - ) - distance = geodinv[2] - - time_to_reach = timedelta(seconds=distance / ship_speed * 3600 / 1852) - arrival_time = time + time_to_reach - - if wp_next.time is None: - time = arrival_time - elif arrival_time > wp_next.time: - raise ScheduleError( - f"Waypoint planning is not valid: would arrive too late at waypoint number {wp_i + 2}. " - f"location: {wp_next.location} time: {wp_next.time} instrument: {wp_next.instrument}" - ) - else: - time = wp_next.time - - print("... All good to go!") - - -def _is_on_land_zero_uv(fieldset: FieldSet, waypoint: Waypoint) -> bool: - """ - Check if waypoint is on land by assuming zero velocity means land. - - :param fieldset: The fieldset to sample the velocity from. - :param waypoint: The waypoint to check. - :returns: If the waypoint is on land. - """ - return fieldset.UV.eval( - 0, - fieldset.gridset.grids[0].depth[0], - waypoint.location.lat, - waypoint.location.lon, - applyConversion=False, - ) == (0.0, 0.0) diff --git a/src/virtualship/models/ship_config.py b/src/virtualship/models/ship_config.py deleted file mode 100644 index ba7d221f0..000000000 --- a/src/virtualship/models/ship_config.py +++ /dev/null @@ -1,310 +0,0 @@ -"""ShipConfig and supporting classes.""" - -from __future__ import annotations - -from datetime import timedelta -from pathlib import Path -from typing import TYPE_CHECKING - -import pydantic -import yaml - -from virtualship.errors import ConfigError -from virtualship.instruments.master import InstrumentType -from virtualship.utils import _validate_numeric_mins_to_timedelta - -if TYPE_CHECKING: - from .schedule import Schedule - - -class ArgoFloatConfig(pydantic.BaseModel): - """Configuration for argos floats.""" - - min_depth_meter: float = pydantic.Field(le=0.0) - max_depth_meter: float = pydantic.Field(le=0.0) - drift_depth_meter: float = pydantic.Field(le=0.0) - vertical_speed_meter_per_second: float = pydantic.Field(lt=0.0) - cycle_days: float = pydantic.Field(gt=0.0) - drift_days: float = pydantic.Field(gt=0.0) - - -class ADCPConfig(pydantic.BaseModel): - """Configuration for ADCP instrument.""" - - max_depth_meter: float = pydantic.Field(le=0.0) - num_bins: int = pydantic.Field(gt=0.0) - period: timedelta = pydantic.Field( - serialization_alias="period_minutes", - validation_alias="period_minutes", - gt=timedelta(), - ) - - model_config = pydantic.ConfigDict(populate_by_name=True) - - @pydantic.field_serializer("period") - def _serialize_period(self, value: timedelta, _info): - return value.total_seconds() / 60.0 - - @pydantic.field_validator("period", mode="before") - def _validate_period(cls, value: int | float | timedelta) -> timedelta: - return _validate_numeric_mins_to_timedelta(value) - - -class CTDConfig(pydantic.BaseModel): - """Configuration for CTD instrument.""" - - stationkeeping_time: timedelta = pydantic.Field( - serialization_alias="stationkeeping_time_minutes", - validation_alias="stationkeeping_time_minutes", - gt=timedelta(), - ) - min_depth_meter: float = pydantic.Field(le=0.0) - max_depth_meter: float = pydantic.Field(le=0.0) - - model_config = pydantic.ConfigDict(populate_by_name=True) - - @pydantic.field_serializer("stationkeeping_time") - def _serialize_stationkeeping_time(self, value: timedelta, _info): - return value.total_seconds() / 60.0 - - @pydantic.field_validator("stationkeeping_time", mode="before") - def _validate_stationkeeping_time(cls, value: int | float | timedelta) -> timedelta: - return _validate_numeric_mins_to_timedelta(value) - - -class CTD_BGCConfig(pydantic.BaseModel): - """Configuration for CTD_BGC instrument.""" - - stationkeeping_time: timedelta = pydantic.Field( - serialization_alias="stationkeeping_time_minutes", - validation_alias="stationkeeping_time_minutes", - gt=timedelta(), - ) - min_depth_meter: float = pydantic.Field(le=0.0) - max_depth_meter: float = pydantic.Field(le=0.0) - - model_config = pydantic.ConfigDict(populate_by_name=True) - - @pydantic.field_serializer("stationkeeping_time") - def _serialize_stationkeeping_time(self, value: timedelta, _info): - return value.total_seconds() / 60.0 - - @pydantic.field_validator("stationkeeping_time", mode="before") - def _validate_stationkeeping_time(cls, value: int | float | timedelta) -> timedelta: - return _validate_numeric_mins_to_timedelta(value) - - -class ShipUnderwaterSTConfig(pydantic.BaseModel): - """Configuration for underwater ST.""" - - period: timedelta = pydantic.Field( - serialization_alias="period_minutes", - validation_alias="period_minutes", - gt=timedelta(), - ) - - model_config = pydantic.ConfigDict(populate_by_name=True) - - @pydantic.field_serializer("period") - def _serialize_period(self, value: timedelta, _info): - return value.total_seconds() / 60.0 - - @pydantic.field_validator("period", mode="before") - def _validate_period(cls, value: int | float | timedelta) -> timedelta: - return _validate_numeric_mins_to_timedelta(value) - - -class DrifterConfig(pydantic.BaseModel): - """Configuration for drifters.""" - - depth_meter: float = pydantic.Field(le=0.0) - lifetime: timedelta = pydantic.Field( - serialization_alias="lifetime_minutes", - validation_alias="lifetime_minutes", - gt=timedelta(), - ) - - model_config = pydantic.ConfigDict(populate_by_name=True) - - @pydantic.field_serializer("lifetime") - def _serialize_lifetime(self, value: timedelta, _info): - return value.total_seconds() / 60.0 - - @pydantic.field_validator("lifetime", mode="before") - def _validate_lifetime(cls, value: int | float | timedelta) -> timedelta: - return _validate_numeric_mins_to_timedelta(value) - - -class XBTConfig(pydantic.BaseModel): - """Configuration for xbt instrument.""" - - min_depth_meter: float = pydantic.Field(le=0.0) - max_depth_meter: float = pydantic.Field(le=0.0) - fall_speed_meter_per_second: float = pydantic.Field(gt=0.0) - deceleration_coefficient: float = pydantic.Field(gt=0.0) - - -class ShipConfig(pydantic.BaseModel): - """Configuration of the virtual ship.""" - - ship_speed_knots: float = pydantic.Field(gt=0.0) - """ - Velocity of the ship in knots. - """ - - argo_float_config: ArgoFloatConfig | None = None - """ - Argo float configuration. - - If None, no argo floats can be deployed. - """ - - adcp_config: ADCPConfig | None = None - """ - ADCP configuration. - - If None, no ADCP measurements will be performed. - """ - - ctd_config: CTDConfig | None = None - """ - CTD configuration. - - If None, no CTDs can be cast. - """ - - ctd_bgc_config: CTD_BGCConfig | None = None - """ - CTD_BGC configuration. - - If None, no BGC CTDs can be cast. - """ - - ship_underwater_st_config: ShipUnderwaterSTConfig | None = None - """ - Ship underwater salinity temperature measurementconfiguration. - - If None, no ST measurements will be performed. - """ - - drifter_config: DrifterConfig | None = None - """ - Drifter configuration. - - If None, no drifters can be deployed. - """ - - xbt_config: XBTConfig | None = None - """ - XBT configuration. - - If None, no XBTs can be cast. - """ - - model_config = pydantic.ConfigDict(extra="forbid") - - def to_yaml(self, file_path: str | Path) -> None: - """ - Write config to yaml file. - - :param file_path: Path to the file to write to. - """ - with open(file_path, "w") as file: - yaml.dump(self.model_dump(by_alias=True), file) - - @classmethod - def from_yaml(cls, file_path: str | Path) -> ShipConfig: - """ - Load config from yaml file. - - :param file_path: Path to the file to load from. - :returns: The config. - """ - with open(file_path) as file: - data = yaml.safe_load(file) - return ShipConfig(**data) - - def verify(self, schedule: Schedule) -> None: - """ - Verify the ship configuration against the provided schedule. - - This function performs two main tasks: - 1. Removes instrument configurations that are not present in the schedule. - 2. Verifies that all instruments in the schedule have corresponding configurations. - - Parameters - ---------- - schedule : Schedule - The schedule object containing the planned instruments and waypoints. - - Returns - ------- - None - - Raises - ------ - ConfigError - If an instrument in the schedule does not have a corresponding configuration. - - Notes - ----- - - Prints a message if a configuration is provided for an instrument not in the schedule. - - Sets the configuration to None for instruments not in the schedule. - - Raises a ConfigError for each instrument in the schedule that lacks a configuration. - - """ - instruments_in_schedule = schedule.get_instruments() - - for instrument in [ - "ARGO_FLOAT", - "DRIFTER", - "XBT", - "CTD", - "CTD_BGC", - ]: # TODO make instrument names consistent capitals or lowercase throughout codebase - if hasattr(self, instrument.lower() + "_config") and not any( - instrument == schedule_instrument.name - for schedule_instrument in instruments_in_schedule - ): - print(f"{instrument} configuration provided but not in schedule.") - setattr(self, instrument.lower() + "_config", None) - - # verify instruments in schedule have configuration - # TODO: the ConfigError message could be improved to explain that the **schedule** file has X instrument but the **ship_config** file does not - for instrument in instruments_in_schedule: - try: - InstrumentType(instrument) - except ValueError as e: - raise NotImplementedError("Instrument not supported.") from e - - if instrument == InstrumentType.ARGO_FLOAT and ( - not hasattr(self, "argo_float_config") or self.argo_float_config is None - ): - raise ConfigError( - "Planning has a waypoint with Argo float instrument, but configuration does not configure Argo floats." - ) - if instrument == InstrumentType.CTD and ( - not hasattr(self, "ctd_config") or self.ctd_config is None - ): - raise ConfigError( - "Planning has a waypoint with CTD instrument, but configuration does not configure CTDs." - ) - if instrument == InstrumentType.CTD_BGC and ( - not hasattr(self, "ctd_bgc_config") or self.ctd_bgc_config is None - ): - raise ConfigError( - "Planning has a waypoint with CTD_BGC instrument, but configuration does not configure CTD_BGCs." - ) - if instrument == InstrumentType.DRIFTER and ( - not hasattr(self, "drifter_config") or self.drifter_config is None - ): - raise ConfigError( - "Planning has a waypoint with drifter instrument, but configuration does not configure drifters." - ) - - if instrument == InstrumentType.XBT and ( - not hasattr(self, "xbt_config") or self.xbt_config is None - ): - raise ConfigError( - "Planning has a waypoint with XBT instrument, but configuration does not configure XBT." - ) From 43c855d9536ef648ce07fa86d22077aad1870fe1 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 22 Oct 2025 15:23:59 +0200 Subject: [PATCH 17/97] add is_underway property to InstrumentType and filter instruments in plan UI --- src/virtualship/cli/_plan.py | 6 +++--- src/virtualship/instruments/master.py | 5 +++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/virtualship/cli/_plan.py b/src/virtualship/cli/_plan.py index 1aa3208bf..435d5ab0d 100644 --- a/src/virtualship/cli/_plan.py +++ b/src/virtualship/cli/_plan.py @@ -29,6 +29,7 @@ type_to_textual, ) from virtualship.errors import UnexpectedError, UserError +from virtualship.instruments.master import InstrumentType from virtualship.models import ( ADCPConfig, ArgoFloatConfig, @@ -36,7 +37,6 @@ CTDConfig, DrifterConfig, Expedition, - InstrumentType, Location, ShipConfig, ShipUnderwaterSTConfig, @@ -630,7 +630,7 @@ def _update_schedule(self): 0, ) wp.instrument = [] - for instrument in InstrumentType: + for instrument in [inst for inst in InstrumentType if not inst.is_underway]: switch_on = self.query_one(f"#wp{i}_{instrument.value}").value if instrument.value == "DRIFTER" and switch_on: count_str = self.query_one(f"#wp{i}_drifter_count").value @@ -901,7 +901,7 @@ def compose(self) -> ComposeResult: ) yield Label("Instruments:") - for instrument in InstrumentType: + for instrument in [i for i in InstrumentType if not i.is_underway]: is_selected = instrument in (self.waypoint.instrument or []) with Horizontal(): yield Label(instrument.value) diff --git a/src/virtualship/instruments/master.py b/src/virtualship/instruments/master.py index cb31468f1..6e0989889 100644 --- a/src/virtualship/instruments/master.py +++ b/src/virtualship/instruments/master.py @@ -14,6 +14,11 @@ class InstrumentType(Enum): ADCP = "ADCP" UNDERWATER_ST = "UNDERWATER_ST" + @property + def is_underway(self) -> bool: + """Return True if instrument is an underway instrument (ADCP, UNDERWATER_ST).""" + return self in {InstrumentType.ADCP, InstrumentType.UNDERWATER_ST} + def get_instruments_registry(): # local imports to avoid circular import issues From 79c81cb69ea8e25c669f8b4e9a58f16e95572504 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 22 Oct 2025 15:24:59 +0200 Subject: [PATCH 18/97] enhance CLI output for fetching --- src/virtualship/cli/_fetch.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/virtualship/cli/_fetch.py b/src/virtualship/cli/_fetch.py index c460db379..1cf33fa94 100644 --- a/src/virtualship/cli/_fetch.py +++ b/src/virtualship/cli/_fetch.py @@ -87,7 +87,7 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None ) shutil.copyfile(path / EXPEDITION, download_folder / EXPEDITION) - # TODO: enhance CLI output for users? + click.echo(f"\n\n{(' Fetching data for: Bathymetry ').center(80, '=')}\n\n") # bathymetry (for all expeditions) copernicusmarine.subset( @@ -116,11 +116,14 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None # iterate across instruments and download data based on space_time_region for itype, instrument in filter_instruments.items(): + click.echo( + f"\n\n{(' Fetching data for: ' + itype.value + ' ').center(80, '=')}\n\n" + ) try: input_dataset = instrument["input_class"]( data_dir=download_folder, credentials=credentials, - space_time_region=expedition.space_time_region, + space_time_region=space_time_region, ) input_dataset.download_data() From efb53cada0ac63e87f8018aa1e8c0353a41039f7 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 22 Oct 2025 15:25:52 +0200 Subject: [PATCH 19/97] general fixes and new error class --- src/virtualship/errors.py | 8 +++++++- src/virtualship/expedition/checkpoint.py | 3 ++- src/virtualship/expedition/do_expedition.py | 2 +- src/virtualship/expedition/simulate_schedule.py | 2 +- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/virtualship/errors.py b/src/virtualship/errors.py index cdd583493..3ba52a9a2 100644 --- a/src/virtualship/errors.py +++ b/src/virtualship/errors.py @@ -22,7 +22,7 @@ class ScheduleError(RuntimeError): pass -class ConfigError(RuntimeError): +class InstrumentsConfigError(RuntimeError): """An error in the config.""" pass @@ -38,3 +38,9 @@ class UnexpectedError(Exception): """Error raised when there is an unexpected problem.""" pass + + +class UnderwayConfigsError(Exception): + """Error raised when underway instrument configurations (ADCP or underwater ST) are missing.""" + + pass diff --git a/src/virtualship/expedition/checkpoint.py b/src/virtualship/expedition/checkpoint.py index 6daf1a9b2..ff2dadd60 100644 --- a/src/virtualship/expedition/checkpoint.py +++ b/src/virtualship/expedition/checkpoint.py @@ -8,7 +8,8 @@ import yaml from virtualship.errors import CheckpointError -from virtualship.models import InstrumentType, Schedule +from virtualship.instruments.master import InstrumentType +from virtualship.models import Schedule class _YamlDumper(yaml.SafeDumper): diff --git a/src/virtualship/expedition/do_expedition.py b/src/virtualship/expedition/do_expedition.py index 5c46d2eb8..921ea528d 100644 --- a/src/virtualship/expedition/do_expedition.py +++ b/src/virtualship/expedition/do_expedition.py @@ -40,7 +40,7 @@ def do_expedition(expedition_dir: str | Path, input_data: Path | None = None) -> expedition = _get_expedition(expedition_dir) # Verify instruments_config file is consistent with schedule - expedition.instruments_config.verify(expedition.schedule) + expedition.instruments_config.verify(expedition) # load last checkpoint checkpoint = _load_checkpoint(expedition_dir) diff --git a/src/virtualship/expedition/simulate_schedule.py b/src/virtualship/expedition/simulate_schedule.py index 3b78c5c72..784e2d328 100644 --- a/src/virtualship/expedition/simulate_schedule.py +++ b/src/virtualship/expedition/simulate_schedule.py @@ -11,10 +11,10 @@ from virtualship.instruments.ctd import CTD from virtualship.instruments.ctd_bgc import CTD_BGC from virtualship.instruments.drifter import Drifter +from virtualship.instruments.master import InstrumentType from virtualship.instruments.xbt import XBT from virtualship.models import ( Expedition, - InstrumentType, Location, Spacetime, Waypoint, From c4ddea1aeadef5ecba43abfe812eae744aa8518a Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 22 Oct 2025 15:26:16 +0200 Subject: [PATCH 20/97] refactor test cases to use Expedition object --- tests/expedition/test_expedition.py | 140 +++++++++++++++++----------- 1 file changed, 85 insertions(+), 55 deletions(-) diff --git a/tests/expedition/test_expedition.py b/tests/expedition/test_expedition.py index a4643e03a..4bed35e70 100644 --- a/tests/expedition/test_expedition.py +++ b/tests/expedition/test_expedition.py @@ -4,9 +4,14 @@ import pyproj import pytest -from virtualship.errors import ConfigError, ScheduleError +from virtualship.errors import InstrumentsConfigError, ScheduleError from virtualship.expedition.do_expedition import _load_input_data -from virtualship.models import Expedition, Location, Schedule, Waypoint +from virtualship.models import ( + Expedition, + Location, + Schedule, + Waypoint, +) from virtualship.utils import EXPEDITION, _get_expedition, get_example_expedition projection = pyproj.Geod(ellps="WGS84") @@ -56,6 +61,7 @@ def test_verify_schedule() -> None: def test_get_instruments() -> None: + get_expedition = _get_expedition(expedition_dir) schedule = Schedule( waypoints=[ Waypoint(location=Location(0, 0), instrument=["CTD"]), @@ -63,12 +69,21 @@ def test_get_instruments() -> None: Waypoint(location=Location(1, 0), instrument=["CTD"]), ] ) - - assert set(instrument.name for instrument in schedule.get_instruments()) == { - "CTD", - "XBT", - "ARGO_FLOAT", - } + expedition = Expedition( + schedule=schedule, + instruments_config=get_expedition.instruments_config, + ship_config=get_expedition.ship_config, + ) + assert ( + set(instrument.name for instrument in expedition.get_instruments()) + == { + "CTD", + "UNDERWATER_ST", # not added above but underway instruments are auto present from instruments_config in expedition_dir/expedition.yaml + "ADCP", # as above + "ARGO_FLOAT", + "XBT", + } + ) @pytest.mark.parametrize( @@ -165,15 +180,15 @@ def test_verify_schedule_errors( @pytest.fixture -def schedule(tmp_file): +def expedition(tmp_file): with open(tmp_file, "w") as file: file.write(get_example_expedition()) - return Expedition.from_yaml(tmp_file).schedule + return Expedition.from_yaml(tmp_file) @pytest.fixture -def schedule_no_xbt(schedule): - for waypoint in schedule.waypoints: +def expedition_no_xbt(expedition): + for waypoint in expedition.schedule.waypoints: if waypoint.instrument and any( instrument.name == "XBT" for instrument in waypoint.instrument ): @@ -183,54 +198,57 @@ def schedule_no_xbt(schedule): if instrument.name != "XBT" ] - return schedule + return expedition @pytest.fixture -def instruments_config(tmp_file): - with open(tmp_file, "w") as file: - file.write(get_example_expedition()) - return Expedition.from_yaml(tmp_file).instruments_config +def instruments_config_no_xbt(expedition): + delattr(expedition.instruments_config, "xbt_config") + return expedition.instruments_config @pytest.fixture -def instruments_config_no_xbt(instruments_config): - delattr(instruments_config, "xbt_config") - return instruments_config +def instruments_config_no_ctd(expedition): + delattr(expedition.instruments_config, "ctd_config") + return expedition.instruments_config @pytest.fixture -def instruments_config_no_ctd(instruments_config): - delattr(instruments_config, "ctd_config") - return instruments_config +def instruments_config_no_ctd_bgc(expedition): + delattr(expedition.instruments_config, "ctd_bgc_config") + return expedition.instruments_config @pytest.fixture -def instruments_config_no_ctd_bgc(instruments_config): - delattr(instruments_config, "ctd_bgc_config") - return instruments_config +def instruments_config_no_argo_float(expedition): + delattr(expedition.instruments_config, "argo_float_config") + return expedition.instruments_config @pytest.fixture -def instruments_config_no_argo_float(instruments_config): - delattr(instruments_config, "argo_float_config") - return instruments_config +def instruments_config_no_drifter(expedition): + delattr(expedition.instruments_config, "drifter_config") + return expedition.instruments_config @pytest.fixture -def instruments_config_no_drifter(instruments_config): - delattr(instruments_config, "drifter_config") - return instruments_config +def instruments_config_no_adcp(expedition): + delattr(expedition.instruments_config, "adcp_config") + return expedition.instruments_config -def test_verify_instruments_config(instruments_config, schedule) -> None: - instruments_config.verify(schedule) +@pytest.fixture +def instruments_config_no_underwater_st(expedition): + delattr(expedition.instruments_config, "ship_underwater_st_config") + return expedition.instruments_config -def test_verify_instruments_config_no_instrument( - instruments_config, schedule_no_xbt -) -> None: - instruments_config.verify(schedule_no_xbt) +def test_verify_instruments_config(expedition) -> None: + expedition.instruments_config.verify(expedition) + + +def test_verify_instruments_config_no_instrument(expedition, expedition_no_xbt) -> None: + expedition.instruments_config.verify(expedition_no_xbt) @pytest.mark.parametrize( @@ -238,40 +256,52 @@ def test_verify_instruments_config_no_instrument( [ pytest.param( "instruments_config_no_xbt", - ConfigError, - "Schedule includes instrument 'XBT', but instruments_config does not provide configuration for it.", - id="ShipConfigNoXBT", + InstrumentsConfigError, + "Expedition includes instrument 'XBT', but instruments_config does not provide configuration for it.", + id="InstrumentsConfigNoXBT", ), pytest.param( "instruments_config_no_ctd", - ConfigError, - "Schedule includes instrument 'CTD', but instruments_config does not provide configuration for it.", - id="ShipConfigNoCTD", + InstrumentsConfigError, + "Expedition includes instrument 'CTD', but instruments_config does not provide configuration for it.", + id="InstrumentsConfigNoCTD", ), pytest.param( "instruments_config_no_ctd_bgc", - ConfigError, - "Schedule includes instrument 'CTD_BGC', but instruments_config does not provide configuration for it.", - id="ShipConfigNoCTD_BGC", + InstrumentsConfigError, + "Expedition includes instrument 'CTD_BGC', but instruments_config does not provide configuration for it.", + id="InstrumentsConfigNoCTD_BGC", ), pytest.param( "instruments_config_no_argo_float", - ConfigError, - "Schedule includes instrument 'ARGO_FLOAT', but instruments_config does not provide configuration for it.", - id="ShipConfigNoARGO_FLOAT", + InstrumentsConfigError, + "Expedition includes instrument 'ARGO_FLOAT', but instruments_config does not provide configuration for it.", + id="InstrumentsConfigNoARGO_FLOAT", ), pytest.param( "instruments_config_no_drifter", - ConfigError, - "Schedule includes instrument 'DRIFTER', but instruments_config does not provide configuration for it.", - id="ShipConfigNoDRIFTER", + InstrumentsConfigError, + "Expedition includes instrument 'DRIFTER', but instruments_config does not provide configuration for it.", + id="InstrumentsConfigNoDRIFTER", + ), + pytest.param( + "instruments_config_no_adcp", + InstrumentsConfigError, + r"Underway instrument config attribute\(s\) are missing from YAML\. Must be Config object or None\.", + id="InstrumentsConfigNoADCP", + ), + pytest.param( + "instruments_config_no_underwater_st", + InstrumentsConfigError, + r"Underway instrument config attribute\(s\) are missing from YAML\. Must be Config object or None\.", + id="InstrumentsConfigNoUNDERWATER_ST", ), ], ) def test_verify_instruments_config_errors( - request, schedule, instruments_config_fixture, error, match + request, expedition, instruments_config_fixture, error, match ) -> None: instruments_config = request.getfixturevalue(instruments_config_fixture) with pytest.raises(error, match=match): - instruments_config.verify(schedule) + instruments_config.verify(expedition) From 66aa4a52347fe7762b007d132f70daa9efd40545 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 27 Oct 2025 16:35:13 +0100 Subject: [PATCH 21/97] move instruments base classes out of models/ dir --- src/virtualship/instruments/adcp.py | 5 +- src/virtualship/instruments/ctd.py | 2 +- src/virtualship/instruments/master.py | 113 +++++++++++++++++++++++++ src/virtualship/models/instruments.py | 115 -------------------------- 4 files changed, 115 insertions(+), 120 deletions(-) delete mode 100644 src/virtualship/models/instruments.py diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index ccf487389..cf5af1690 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -3,14 +3,11 @@ from typing import ClassVar import numpy as np -from parcels import FieldSet, ParticleSet, ScipyParticle, Variable +from parcels import FieldSet, ParticleSet, ScipyParticle, Variable from virtualship.models.instruments import InputDataset from virtualship.models.spacetime import Spacetime -## TODO: __init__.py will also need updating! -# + therefore instructions for adding new instruments will also involve adding to __init__.py as well as the new instrument script + update InstrumentType in instruments.py - @dataclass class ADCP: diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 15a462349..73125838d 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -4,8 +4,8 @@ from typing import ClassVar import numpy as np -from parcels import FieldSet, JITParticle, ParticleSet, Variable +from parcels import FieldSet, JITParticle, ParticleSet, Variable from virtualship.models.instruments import InputDataset from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/master.py b/src/virtualship/instruments/master.py index 6e0989889..43fe6a00c 100644 --- a/src/virtualship/instruments/master.py +++ b/src/virtualship/instruments/master.py @@ -1,4 +1,14 @@ +import abc +from collections.abc import Callable +from datetime import timedelta from enum import Enum +from pathlib import Path + +import copernicusmarine +from yaspin import yaspin + +from virtualship.models.space_time_region import SpaceTimeRegion +from virtualship.utils import ship_spinner class InstrumentType(Enum): @@ -47,3 +57,106 @@ def get_instruments_registry(): for inst in InstrumentType if _input_class_map.get(inst.value) is not None } + + +# Base classes + + +class InputDataset(abc.ABC): + """Base class for instrument input datasets.""" + + def __init__( + self, + name: str, + latlon_buffer: float, + datetime_buffer: float, + min_depth: float, + max_depth: float, + data_dir: str, + credentials: dict, + space_time_region: SpaceTimeRegion, + ): + """Initialise input dataset.""" + self.name = name + self.latlon_buffer = latlon_buffer + self.datetime_buffer = datetime_buffer + self.min_depth = min_depth + self.max_depth = max_depth + self.data_dir = data_dir + self.credentials = credentials + self.space_time_region = space_time_region + + @abc.abstractmethod + def get_datasets_dict(self) -> dict: + """Get parameters for instrument's variable(s) specific data download.""" + ... + + def download_data(self) -> None: + """Download data for the instrument using copernicusmarine.""" + parameter_args = dict( + minimum_longitude=self.space_time_region.spatial_range.minimum_longitude + - self.latlon_buffer, + maximum_longitude=self.space_time_region.spatial_range.maximum_longitude + + self.latlon_buffer, + minimum_latitude=self.space_time_region.spatial_range.minimum_latitude + - self.latlon_buffer, + maximum_latitude=self.space_time_region.spatial_range.maximum_latitude + + self.latlon_buffer, + start_datetime=self.space_time_region.time_range.start_time, + end_datetime=self.space_time_region.time_range.end_time + + timedelta(days=self.datetime_buffer), + minimum_depth=abs(self.min_depth), + maximum_depth=abs(self.max_depth), + output_directory=self.data_dir, + username=self.credentials["username"], + password=self.credentials["password"], + overwrite=True, + coordinates_selection_method="outside", + ) + + datasets_args = self.get_datasets_dict() + + for dataset in datasets_args.values(): + download_args = {**parameter_args, **dataset} + copernicusmarine.subset(**download_args) + + +class Instrument(abc.ABC): + """Base class for instruments.""" + + def __init__( + self, + name: str, + config, + input_dataset: InputDataset, + kernels: list[Callable], + ): + """Initialise instrument.""" + self.name = name + self.config = config + self.input_data = input_dataset + self.kernels = kernels + + # def load_fieldset(self): + # """Load fieldset for simulation.""" + # # paths = self.input_data.get_fieldset_paths() + # ... + + def get_output_path(self, output_dir: Path) -> Path: + """Get output path for results.""" + return output_dir / f"{self.name}.zarr" + + def run(self): + """Run instrument simulation.""" + with yaspin( + text=f"Simulating {self.name} measurements... ", + side="right", + spinner=ship_spinner, + ) as spinner: + self.simulate() + spinner.ok("✅") + + @abc.abstractmethod + def simulate(self): + """Simulate instrument measurements.""" + ... diff --git a/src/virtualship/models/instruments.py b/src/virtualship/models/instruments.py deleted file mode 100644 index 72d9237de..000000000 --- a/src/virtualship/models/instruments.py +++ /dev/null @@ -1,115 +0,0 @@ -import abc -from collections.abc import Callable -from datetime import timedelta -from pathlib import Path - -import copernicusmarine -from yaspin import yaspin - -from virtualship.models.space_time_region import SpaceTimeRegion -from virtualship.utils import ship_spinner - -# TODO: -# Discussion: Should each instrument manage its own data files for modularity, -# or should we consolidate downloads to minimize file duplication across instruments? -# Consider starting with per-instrument files for simplicity, and refactor later if needed. - - -class InputDataset(abc.ABC): - """Base class for instrument input datasets.""" - - def __init__( - self, - name: str, - latlon_buffer: float, - datetime_buffer: float, - min_depth: float, - max_depth: float, - data_dir: str, - credentials: dict, - space_time_region: SpaceTimeRegion, - ): - """Initialise input dataset.""" - self.name = name - self.latlon_buffer = latlon_buffer - self.datetime_buffer = datetime_buffer - self.min_depth = min_depth - self.max_depth = max_depth - self.data_dir = data_dir - self.credentials = credentials - self.space_time_region = space_time_region - - @abc.abstractmethod - def get_datasets_dict(self) -> dict: - """Get parameters for instrument's variable(s) specific data download.""" - ... - - def download_data(self) -> None: - """Download data for the instrument using copernicusmarine.""" - parameter_args = dict( - minimum_longitude=self.space_time_region.spatial_range.minimum_longitude - - self.latlon_buffer, - maximum_longitude=self.space_time_region.spatial_range.maximum_longitude - + self.latlon_buffer, - minimum_latitude=self.space_time_region.spatial_range.minimum_latitude - - self.latlon_buffer, - maximum_latitude=self.space_time_region.spatial_range.maximum_latitude - + self.latlon_buffer, - start_datetime=self.space_time_region.time_range.start_time, - end_datetime=self.space_time_region.time_range.end_time - + timedelta(days=self.datetime_buffer), - minimum_depth=abs(self.min_depth), - maximum_depth=abs(self.max_depth), - output_directory=self.data_dir, - username=self.credentials["username"], - password=self.credentials["password"], - overwrite=True, - coordinates_selection_method="outside", - ) - - datasets_args = self.get_datasets_dict() - - for dataset in datasets_args.values(): - download_args = {**parameter_args, **dataset} - copernicusmarine.subset(**download_args) - - -class Instrument(abc.ABC): - """Base class for instruments.""" - - def __init__( - self, - name: str, - config, - input_dataset: InputDataset, - kernels: list[Callable], - ): - """Initialise instrument.""" - self.name = name - self.config = config - self.input_data = input_dataset - self.kernels = kernels - - # def load_fieldset(self): - # """Load fieldset for simulation.""" - # # paths = self.input_data.get_fieldset_paths() - # ... - - def get_output_path(self, output_dir: Path) -> Path: - """Get output path for results.""" - return output_dir / f"{self.name}.zarr" - - def run(self): - """Run instrument simulation.""" - with yaspin( - text=f"Simulating {self.name} measurements... ", - side="right", - spinner=ship_spinner, - ) as spinner: - self.simulate() - spinner.ok("✅") - - @abc.abstractmethod - def simulate(self): - """Simulate instrument measurements.""" - ... From 06ddf37e636775bfc01fa9201d749310b50c96e4 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 27 Oct 2025 16:35:53 +0100 Subject: [PATCH 22/97] update base class imports --- src/virtualship/instruments/adcp.py | 2 +- src/virtualship/instruments/argo_float.py | 4 +- src/virtualship/instruments/ctd.py | 2 +- src/virtualship/instruments/ctd_bgc.py | 2 +- src/virtualship/instruments/drifter.py | 4 +- .../instruments/ship_underwater_st.py | 4 +- src/virtualship/instruments/xbt.py | 4 +- src/virtualship/models/expedition.py | 12 ++- tests/models/test_instruments.py | 97 ------------------- 9 files changed, 20 insertions(+), 111 deletions(-) delete mode 100644 tests/models/test_instruments.py diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index cf5af1690..b80bb544a 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -5,7 +5,7 @@ import numpy as np from parcels import FieldSet, ParticleSet, ScipyParticle, Variable -from virtualship.models.instruments import InputDataset +from virtualship.instruments.master import InputDataset from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index b9949fb16..e9cadb8d8 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -5,6 +5,7 @@ from typing import ClassVar import numpy as np + from parcels import ( AdvectionRK4, FieldSet, @@ -13,8 +14,7 @@ StatusCode, Variable, ) - -from virtualship.models.instruments import InputDataset +from virtualship.instruments.master import InputDataset from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 73125838d..6a27d175c 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -6,7 +6,7 @@ import numpy as np from parcels import FieldSet, JITParticle, ParticleSet, Variable -from virtualship.models.instruments import InputDataset +from virtualship.instruments.master import InputDataset from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 96c0efa79..74b5b81c6 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -6,7 +6,7 @@ import numpy as np from parcels import FieldSet, JITParticle, ParticleSet, Variable -from virtualship.models.instruments import InputDataset +from virtualship.instruments.master import InputDataset from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index 60ba5558a..40aad9d42 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -4,9 +4,9 @@ from typing import ClassVar import numpy as np -from parcels import AdvectionRK4, FieldSet, JITParticle, ParticleSet, Variable -from virtualship.models.instruments import InputDataset +from parcels import AdvectionRK4, FieldSet, JITParticle, ParticleSet, Variable +from virtualship.instruments.master import InputDataset from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index 332c0b2c3..3c7bc0ee5 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -3,9 +3,9 @@ from typing import ClassVar import numpy as np -from parcels import FieldSet, ParticleSet, ScipyParticle, Variable -from virtualship.models.instruments import InputDataset +from parcels import FieldSet, ParticleSet, ScipyParticle, Variable +from virtualship.instruments.master import InputDataset from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index 81cba7b90..4351d08e9 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -4,9 +4,9 @@ from typing import ClassVar import numpy as np -from parcels import FieldSet, JITParticle, ParticleSet, Variable -from virtualship.models.instruments import InputDataset +from parcels import FieldSet, JITParticle, ParticleSet, Variable +from virtualship.instruments.master import InputDataset from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index 7e206e8f1..1b501119d 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -9,7 +9,6 @@ import yaml from virtualship.errors import InstrumentsConfigError, ScheduleError -from virtualship.instruments.master import InstrumentType from virtualship.utils import _validate_numeric_mins_to_timedelta from .location import Location @@ -17,8 +16,8 @@ if TYPE_CHECKING: from parcels import FieldSet - from virtualship.expedition.input_data import InputData + from virtualship.instruments.master import InstrumentType projection: pyproj.Geod = pyproj.Geod(ellps="WGS84") @@ -45,8 +44,10 @@ def from_yaml(cls, file_path: str) -> Expedition: data = yaml.safe_load(file) return Expedition(**data) - def get_instruments(self) -> set[InstrumentType]: + def get_instruments(self): """Return a set of unique InstrumentType enums used in the expedition.""" + from virtualship.instruments.master import InstrumentType + instruments_in_expedition = [] # from waypoints for waypoint in self.schedule.waypoints: @@ -114,6 +115,8 @@ def verify( """ print("\nVerifying route... ") + from virtualship.instruments.master import InstrumentType + if check_space_time_region and self.space_time_region is None: raise ScheduleError( "space_time_region not found in schedule, please define it to fetch the data." @@ -180,6 +183,7 @@ def verify( # check that ship will arrive on time at each waypoint (in case no unexpected event happen) time = self.waypoints[0].time + for wp_i, (wp, wp_next) in enumerate( zip(self.waypoints, self.waypoints[1:], strict=False) ): @@ -414,6 +418,8 @@ def verify(self, expedition: Expedition) -> None: Removes instrument configs not present in the schedule and checks that all scheduled instruments are configured. Raises ConfigError if any scheduled instrument is missing a config. """ + from virtualship.instruments.master import InstrumentType + instruments_in_expedition = expedition.get_instruments() instrument_config_map = { InstrumentType.ARGO_FLOAT: "argo_float_config", diff --git a/tests/models/test_instruments.py b/tests/models/test_instruments.py deleted file mode 100644 index 2ad73cdfc..000000000 --- a/tests/models/test_instruments.py +++ /dev/null @@ -1,97 +0,0 @@ -import datetime -from unittest.mock import patch - -import pytest - -from virtualship.models.instruments import InputDataset -from virtualship.models.space_time_region import ( - SpaceTimeRegion, - SpatialRange, - TimeRange, -) - - -class DummyInputDataset(InputDataset): - """A minimal InputDataset subclass for testing purposes.""" - - def get_datasets_dict(self): - """Return a dummy datasets dict for testing.""" - return { - "dummy": { - "dataset_id": "test_id", - "variables": ["var1"], - "output_filename": "dummy.nc", - } - } - - -@pytest.fixture -def dummy_space_time_region(): - spatial_range = SpatialRange( - minimum_longitude=0, - maximum_longitude=1, - minimum_latitude=0, - maximum_latitude=1, - minimum_depth=0, - maximum_depth=10, - ) - base_time = datetime.datetime.strptime("1950-01-01", "%Y-%m-%d") - time_range = TimeRange( - start_time=base_time, - end_time=base_time + datetime.timedelta(hours=1), - ) - return SpaceTimeRegion( - spatial_range=spatial_range, - time_range=time_range, - ) - - -def test_inputdataset_abstract_instantiation(): - # instantiation should not be allowed - with pytest.raises(TypeError): - InputDataset( - name="test", - latlon_buffer=0, - datetime_buffer=0, - min_depth=0, - max_depth=10, - data_dir=".", - credentials={"username": "u", "password": "p"}, - space_time_region=None, - ) - - -def test_dummyinputdataset_initialization(dummy_space_time_region): - ds = DummyInputDataset( - name="test", - latlon_buffer=0.5, - datetime_buffer=1, - min_depth=0, - max_depth=10, - data_dir=".", - credentials={"username": "u", "password": "p"}, - space_time_region=dummy_space_time_region, - ) - assert ds.name == "test" - assert ds.latlon_buffer == 0.5 - assert ds.datetime_buffer == 1 - assert ds.min_depth == 0 - assert ds.max_depth == 10 - assert ds.data_dir == "." - assert ds.credentials["username"] == "u" - - -@patch("virtualship.models.instruments.copernicusmarine.subset") -def test_download_data_calls_subset(mock_subset, dummy_space_time_region): - ds = DummyInputDataset( - name="test", - latlon_buffer=0.5, - datetime_buffer=1, - min_depth=0, - max_depth=10, - data_dir=".", - credentials={"username": "u", "password": "p"}, - space_time_region=dummy_space_time_region, - ) - ds.download_data() - assert mock_subset.called From 5887177fd9df2e0387a2a5e32d7ede4457196a51 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 27 Oct 2025 16:36:20 +0100 Subject: [PATCH 23/97] make get_instruments_registry more robust with testing --- src/virtualship/instruments/master.py | 4 +- tests/instruments/test_master.py | 109 ++++++++++++++++++++++++++ 2 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 tests/instruments/test_master.py diff --git a/src/virtualship/instruments/master.py b/src/virtualship/instruments/master.py index 43fe6a00c..2139e8d51 100644 --- a/src/virtualship/instruments/master.py +++ b/src/virtualship/instruments/master.py @@ -55,12 +55,14 @@ def get_instruments_registry(): "input_class": _input_class_map.get(inst.value), } for inst in InstrumentType - if _input_class_map.get(inst.value) is not None } # Base classes +# TODO: could InputDataset and Instrument be unified? +# TODO: and all associated child classes... + class InputDataset(abc.ABC): """Base class for instrument input datasets.""" diff --git a/tests/instruments/test_master.py b/tests/instruments/test_master.py new file mode 100644 index 000000000..f84238d09 --- /dev/null +++ b/tests/instruments/test_master.py @@ -0,0 +1,109 @@ +import datetime +from unittest.mock import patch + +import pytest + +from virtualship.instruments.master import ( + InputDataset, + InstrumentType, + get_instruments_registry, +) +from virtualship.models.space_time_region import ( + SpaceTimeRegion, + SpatialRange, + TimeRange, +) + + +class DummyInputDataset(InputDataset): + """A minimal InputDataset subclass for testing purposes.""" + + def get_datasets_dict(self): + """Return a dummy datasets dict for testing.""" + return { + "dummy": { + "dataset_id": "test_id", + "variables": ["var1"], + "output_filename": "dummy.nc", + } + } + + +@pytest.fixture +def dummy_space_time_region(): + spatial_range = SpatialRange( + minimum_longitude=0, + maximum_longitude=1, + minimum_latitude=0, + maximum_latitude=1, + minimum_depth=0, + maximum_depth=10, + ) + base_time = datetime.datetime.strptime("1950-01-01", "%Y-%m-%d") + time_range = TimeRange( + start_time=base_time, + end_time=base_time + datetime.timedelta(hours=1), + ) + return SpaceTimeRegion( + spatial_range=spatial_range, + time_range=time_range, + ) + + +def test_inputdataset_abstract_instantiation(): + # instantiation should not be allowed + with pytest.raises(TypeError): + InputDataset( + name="test", + latlon_buffer=0, + datetime_buffer=0, + min_depth=0, + max_depth=10, + data_dir=".", + credentials={"username": "u", "password": "p"}, + space_time_region=None, + ) + + +def test_dummyinputdataset_initialization(dummy_space_time_region): + ds = DummyInputDataset( + name="test", + latlon_buffer=0.5, + datetime_buffer=1, + min_depth=0, + max_depth=10, + data_dir=".", + credentials={"username": "u", "password": "p"}, + space_time_region=dummy_space_time_region, + ) + assert ds.name == "test" + assert ds.latlon_buffer == 0.5 + assert ds.datetime_buffer == 1 + assert ds.min_depth == 0 + assert ds.max_depth == 10 + assert ds.data_dir == "." + assert ds.credentials["username"] == "u" + + +@patch("virtualship.models.instruments.copernicusmarine.subset") +def test_download_data_calls_subset(mock_subset, dummy_space_time_region): + ds = DummyInputDataset( + name="test", + latlon_buffer=0.5, + datetime_buffer=1, + min_depth=0, + max_depth=10, + data_dir=".", + credentials={"username": "u", "password": "p"}, + space_time_region=dummy_space_time_region, + ) + ds.download_data() + assert mock_subset.called + + +def test_all_instruments_have_input_class(): + registry = get_instruments_registry() + for instrument in InstrumentType: + entry = registry.get(instrument) + assert entry is not None, f"No registry entry for {instrument}" + assert entry.get("input_class") is not None, f"No input_class for {instrument}" From cc35538de40aded688b3e346353f190a0f24417c Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 27 Oct 2025 17:23:40 +0100 Subject: [PATCH 24/97] update mock reanalysis period and refactor tests to use expedition fixture --- tests/cli/test_cli.py | 4 ++-- tests/cli/test_fetch.py | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 3d46787f9..1b2c6e53a 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -23,9 +23,9 @@ def fake_open_dataset(*args, **kwargs): "time": ( "time", [ - np.datetime64("1993-01-01"), np.datetime64("2022-01-01"), - ], # mock up rough renanalysis period + np.datetime64("2025-01-01"), + ], # mock up rough reanalysis period, covers test schedule ) } ) diff --git a/tests/cli/test_fetch.py b/tests/cli/test_fetch.py index 3ea56ea0f..9725601a3 100644 --- a/tests/cli/test_fetch.py +++ b/tests/cli/test_fetch.py @@ -38,8 +38,8 @@ def fake_open_dataset(*args, **kwargs): "time": ( "time", [ - np.datetime64("1993-01-01"), np.datetime64("2022-01-01"), + np.datetime64("2025-01-01"), ], # mock up rough renanalysis period ) } @@ -65,7 +65,7 @@ def expedition(tmpdir): @pytest.mark.usefixtures("copernicus_no_download") -def test_fetch(schedule, ship_config, tmpdir): +def test_fetch(expedition, tmpdir): """Test the fetch command, but mock the download and dataset metadata interrogation.""" _fetch(Path(tmpdir), "test", "test") @@ -99,12 +99,12 @@ def test_complete_download(tmp_path): @pytest.mark.usefixtures("copernicus_no_download") -def test_select_product_id(schedule): +def test_select_product_id(expedition): """Should return the physical reanalysis product id via the timings prescribed in the static schedule.yaml file.""" result = select_product_id( physical=True, - schedule_start=schedule.space_time_region.time_range.start_time, - schedule_end=schedule.space_time_region.time_range.end_time, + schedule_start=expedition.schedule.space_time_region.time_range.start_time, + schedule_end=expedition.schedule.space_time_region.time_range.end_time, username="test", password="test", ) @@ -112,12 +112,12 @@ def test_select_product_id(schedule): @pytest.mark.usefixtures("copernicus_no_download") -def test_start_end_in_product_timerange(schedule): +def test_start_end_in_product_timerange(expedition): """Should return True for valid range ass determined by the static schedule.yaml file.""" assert start_end_in_product_timerange( selected_id="cmems_mod_glo_phy_my_0.083deg_P1D-m", - schedule_start=schedule.space_time_region.time_range.start_time, - schedule_end=schedule.space_time_region.time_range.end_time, + schedule_start=expedition.schedule.space_time_region.time_range.start_time, + schedule_end=expedition.schedule.space_time_region.time_range.end_time, username="test", password="test", ) From 8f5af0466679b8a284f2b76c39c1ecfb4c863854 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 27 Oct 2025 17:41:55 +0100 Subject: [PATCH 25/97] refactor: reorganize instrument classes and update imports for clarity --- src/virtualship/cli/_fetch.py | 36 ++-- src/virtualship/cli/_plan.py | 2 +- src/virtualship/expedition/checkpoint.py | 2 +- .../expedition/simulate_schedule.py | 2 +- src/virtualship/instruments/adcp.py | 2 +- src/virtualship/instruments/argo_float.py | 2 +- src/virtualship/instruments/base.py | 109 ++++++++++++ src/virtualship/instruments/ctd.py | 2 +- src/virtualship/instruments/ctd_bgc.py | 2 +- src/virtualship/instruments/drifter.py | 2 +- src/virtualship/instruments/master.py | 164 ------------------ .../instruments/ship_underwater_st.py | 2 +- src/virtualship/instruments/types.py | 18 ++ src/virtualship/instruments/xbt.py | 2 +- src/virtualship/models/expedition.py | 11 +- 15 files changed, 163 insertions(+), 195 deletions(-) create mode 100644 src/virtualship/instruments/base.py create mode 100644 src/virtualship/instruments/types.py diff --git a/src/virtualship/cli/_fetch.py b/src/virtualship/cli/_fetch.py index 140163a62..17fb403ab 100644 --- a/src/virtualship/cli/_fetch.py +++ b/src/virtualship/cli/_fetch.py @@ -12,7 +12,6 @@ from pydantic import BaseModel from virtualship.errors import CopernicusCatalogueError, IncompleteDownloadError -from virtualship.instruments.master import get_instruments_registry from virtualship.utils import ( _dump_yaml, _generic_load_yaml, @@ -29,8 +28,6 @@ DOWNLOAD_METADATA = "download_metadata.yaml" -INSTRUMENTS = get_instruments_registry() - def _fetch(path: str | Path, username: str | None, password: str | None) -> None: """ @@ -110,29 +107,44 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None coordinates_selection_method="outside", ) - # access instrument classes but keep only instruments which are in schedule - filter_instruments = { - k: v for k, v in INSTRUMENTS.items() if k in instruments_in_expedition + # Direct mapping from InstrumentType to input dataset class + from virtualship.instruments.adcp import ADCPInputDataset + from virtualship.instruments.argo_float import ArgoFloatInputDataset + from virtualship.instruments.ctd import CTDInputDataset + from virtualship.instruments.ctd_bgc import CTD_BGCInputDataset + from virtualship.instruments.drifter import DrifterInputDataset + from virtualship.instruments.ship_underwater_st import Underwater_STInputDataset + from virtualship.instruments.types import InstrumentType + from virtualship.instruments.xbt import XBTInputDataset + + INSTRUMENT_INPUT_DATASET_MAP = { + InstrumentType.CTD: CTDInputDataset, + InstrumentType.CTD_BGC: CTD_BGCInputDataset, + InstrumentType.DRIFTER: DrifterInputDataset, + InstrumentType.ARGO_FLOAT: ArgoFloatInputDataset, + InstrumentType.XBT: XBTInputDataset, + InstrumentType.ADCP: ADCPInputDataset, + InstrumentType.UNDERWATER_ST: Underwater_STInputDataset, } - # iterate across instruments and download data based on space_time_region - for itype, instrument in filter_instruments.items(): + # Only keep instruments present in the expedition + for itype in instruments_in_expedition: + input_dataset_class = INSTRUMENT_INPUT_DATASET_MAP.get(itype) + if input_dataset_class is None: + continue click.echo( f"\n\n{(' Fetching data for: ' + itype.value + ' ').center(80, '=')}\n\n" ) try: - input_dataset = instrument["input_class"]( + input_dataset = input_dataset_class( data_dir=download_folder, credentials=credentials, space_time_region=space_time_region, ) - input_dataset.download_data() - except InvalidUsernameOrPassword as e: shutil.rmtree(download_folder) raise e - click.echo(f"{itype.value} data download completed.") complete_download(download_folder) diff --git a/src/virtualship/cli/_plan.py b/src/virtualship/cli/_plan.py index b6e44a709..a071c38e7 100644 --- a/src/virtualship/cli/_plan.py +++ b/src/virtualship/cli/_plan.py @@ -29,7 +29,7 @@ type_to_textual, ) from virtualship.errors import UnexpectedError, UserError -from virtualship.instruments.master import InstrumentType +from virtualship.instruments.types import InstrumentType from virtualship.models import ( ADCPConfig, ArgoFloatConfig, diff --git a/src/virtualship/expedition/checkpoint.py b/src/virtualship/expedition/checkpoint.py index ff2dadd60..98fe1ae0a 100644 --- a/src/virtualship/expedition/checkpoint.py +++ b/src/virtualship/expedition/checkpoint.py @@ -8,7 +8,7 @@ import yaml from virtualship.errors import CheckpointError -from virtualship.instruments.master import InstrumentType +from virtualship.instruments.types import InstrumentType from virtualship.models import Schedule diff --git a/src/virtualship/expedition/simulate_schedule.py b/src/virtualship/expedition/simulate_schedule.py index 784e2d328..f8d142eac 100644 --- a/src/virtualship/expedition/simulate_schedule.py +++ b/src/virtualship/expedition/simulate_schedule.py @@ -11,7 +11,7 @@ from virtualship.instruments.ctd import CTD from virtualship.instruments.ctd_bgc import CTD_BGC from virtualship.instruments.drifter import Drifter -from virtualship.instruments.master import InstrumentType +from virtualship.instruments.types import InstrumentType from virtualship.instruments.xbt import XBT from virtualship.models import ( Expedition, diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index b80bb544a..078effebd 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -5,7 +5,7 @@ import numpy as np from parcels import FieldSet, ParticleSet, ScipyParticle, Variable -from virtualship.instruments.master import InputDataset +from virtualship.instruments.base import InputDataset from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index e9cadb8d8..c654fd1e1 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -14,7 +14,7 @@ StatusCode, Variable, ) -from virtualship.instruments.master import InputDataset +from virtualship.instruments.base import InputDataset from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py new file mode 100644 index 000000000..dfdb9c6ab --- /dev/null +++ b/src/virtualship/instruments/base.py @@ -0,0 +1,109 @@ +import abc +from collections.abc import Callable +from datetime import timedelta +from pathlib import Path + +import copernicusmarine +import yaspin + +from virtualship.models import SpaceTimeRegion +from virtualship.utils import ship_spinner + + +class InputDataset(abc.ABC): + """Base class for instrument input datasets.""" + + def __init__( + self, + name: str, + latlon_buffer: float, + datetime_buffer: float, + min_depth: float, + max_depth: float, + data_dir: str, + credentials: dict, + space_time_region: SpaceTimeRegion, + ): + """Initialise input dataset.""" + self.name = name + self.latlon_buffer = latlon_buffer + self.datetime_buffer = datetime_buffer + self.min_depth = min_depth + self.max_depth = max_depth + self.data_dir = data_dir + self.credentials = credentials + self.space_time_region = space_time_region + + @abc.abstractmethod + def get_datasets_dict(self) -> dict: + """Get parameters for instrument's variable(s) specific data download.""" + ... + + def download_data(self) -> None: + """Download data for the instrument using copernicusmarine.""" + parameter_args = dict( + minimum_longitude=self.space_time_region.spatial_range.minimum_longitude + - self.latlon_buffer, + maximum_longitude=self.space_time_region.spatial_range.maximum_longitude + + self.latlon_buffer, + minimum_latitude=self.space_time_region.spatial_range.minimum_latitude + - self.latlon_buffer, + maximum_latitude=self.space_time_region.spatial_range.maximum_latitude + + self.latlon_buffer, + start_datetime=self.space_time_region.time_range.start_time, + end_datetime=self.space_time_region.time_range.end_time + + timedelta(days=self.datetime_buffer), + minimum_depth=abs(self.min_depth), + maximum_depth=abs(self.max_depth), + output_directory=self.data_dir, + username=self.credentials["username"], + password=self.credentials["password"], + overwrite=True, + coordinates_selection_method="outside", + ) + + datasets_args = self.get_datasets_dict() + for dataset in datasets_args.values(): + download_args = {**parameter_args, **dataset} + copernicusmarine.subset(**download_args) + + +class Instrument(abc.ABC): + """Base class for instruments.""" + + def __init__( + self, + name: str, + config, + input_dataset: InputDataset, + kernels: list[Callable], + ): + """Initialise instrument.""" + self.name = name + self.config = config + self.input_data = input_dataset + self.kernels = kernels + + # def load_fieldset(self): + # """Load fieldset for simulation.""" + # # paths = self.input_data.get_fieldset_paths() + # ... + + def get_output_path(self, output_dir: Path) -> Path: + """Get output path for results.""" + return output_dir / f"{self.name}.zarr" + + def run(self): + """Run instrument simulation.""" + with yaspin( + text=f"Simulating {self.name} measurements... ", + side="right", + spinner=ship_spinner, + ) as spinner: + self.simulate() + spinner.ok("✅") + + @abc.abstractmethod + def simulate(self): + """Simulate instrument measurements.""" + ... diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 6a27d175c..7d5846011 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -6,7 +6,7 @@ import numpy as np from parcels import FieldSet, JITParticle, ParticleSet, Variable -from virtualship.instruments.master import InputDataset +from virtualship.instruments.base import InputDataset from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 74b5b81c6..1f45ce95f 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -6,7 +6,7 @@ import numpy as np from parcels import FieldSet, JITParticle, ParticleSet, Variable -from virtualship.instruments.master import InputDataset +from virtualship.instruments.base import InputDataset from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index 40aad9d42..df765a0de 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -6,7 +6,7 @@ import numpy as np from parcels import AdvectionRK4, FieldSet, JITParticle, ParticleSet, Variable -from virtualship.instruments.master import InputDataset +from virtualship.instruments.base import InputDataset from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/master.py b/src/virtualship/instruments/master.py index 2139e8d51..e69de29bb 100644 --- a/src/virtualship/instruments/master.py +++ b/src/virtualship/instruments/master.py @@ -1,164 +0,0 @@ -import abc -from collections.abc import Callable -from datetime import timedelta -from enum import Enum -from pathlib import Path - -import copernicusmarine -from yaspin import yaspin - -from virtualship.models.space_time_region import SpaceTimeRegion -from virtualship.utils import ship_spinner - - -class InstrumentType(Enum): - """Types of the instruments.""" - - # TODO: scope for this to evaporate in the future...? - - CTD = "CTD" - CTD_BGC = "CTD_BGC" - DRIFTER = "DRIFTER" - ARGO_FLOAT = "ARGO_FLOAT" - XBT = "XBT" - ADCP = "ADCP" - UNDERWATER_ST = "UNDERWATER_ST" - - @property - def is_underway(self) -> bool: - """Return True if instrument is an underway instrument (ADCP, UNDERWATER_ST).""" - return self in {InstrumentType.ADCP, InstrumentType.UNDERWATER_ST} - - -def get_instruments_registry(): - # local imports to avoid circular import issues - from virtualship.instruments.adcp import ADCPInputDataset - from virtualship.instruments.argo_float import ArgoFloatInputDataset - from virtualship.instruments.ctd import CTDInputDataset - from virtualship.instruments.ctd_bgc import CTD_BGCInputDataset - from virtualship.instruments.drifter import DrifterInputDataset - from virtualship.instruments.ship_underwater_st import Underwater_STInputDataset - from virtualship.instruments.xbt import XBTInputDataset - - _input_class_map = { - "CTD": CTDInputDataset, - "CTD_BGC": CTD_BGCInputDataset, - "DRIFTER": DrifterInputDataset, - "ARGO_FLOAT": ArgoFloatInputDataset, - "XBT": XBTInputDataset, - "ADCP": ADCPInputDataset, - "UNDERWATER_ST": Underwater_STInputDataset, - } - - return { - inst: { - "input_class": _input_class_map.get(inst.value), - } - for inst in InstrumentType - } - - -# Base classes - -# TODO: could InputDataset and Instrument be unified? -# TODO: and all associated child classes... - - -class InputDataset(abc.ABC): - """Base class for instrument input datasets.""" - - def __init__( - self, - name: str, - latlon_buffer: float, - datetime_buffer: float, - min_depth: float, - max_depth: float, - data_dir: str, - credentials: dict, - space_time_region: SpaceTimeRegion, - ): - """Initialise input dataset.""" - self.name = name - self.latlon_buffer = latlon_buffer - self.datetime_buffer = datetime_buffer - self.min_depth = min_depth - self.max_depth = max_depth - self.data_dir = data_dir - self.credentials = credentials - self.space_time_region = space_time_region - - @abc.abstractmethod - def get_datasets_dict(self) -> dict: - """Get parameters for instrument's variable(s) specific data download.""" - ... - - def download_data(self) -> None: - """Download data for the instrument using copernicusmarine.""" - parameter_args = dict( - minimum_longitude=self.space_time_region.spatial_range.minimum_longitude - - self.latlon_buffer, - maximum_longitude=self.space_time_region.spatial_range.maximum_longitude - + self.latlon_buffer, - minimum_latitude=self.space_time_region.spatial_range.minimum_latitude - - self.latlon_buffer, - maximum_latitude=self.space_time_region.spatial_range.maximum_latitude - + self.latlon_buffer, - start_datetime=self.space_time_region.time_range.start_time, - end_datetime=self.space_time_region.time_range.end_time - + timedelta(days=self.datetime_buffer), - minimum_depth=abs(self.min_depth), - maximum_depth=abs(self.max_depth), - output_directory=self.data_dir, - username=self.credentials["username"], - password=self.credentials["password"], - overwrite=True, - coordinates_selection_method="outside", - ) - - datasets_args = self.get_datasets_dict() - - for dataset in datasets_args.values(): - download_args = {**parameter_args, **dataset} - copernicusmarine.subset(**download_args) - - -class Instrument(abc.ABC): - """Base class for instruments.""" - - def __init__( - self, - name: str, - config, - input_dataset: InputDataset, - kernels: list[Callable], - ): - """Initialise instrument.""" - self.name = name - self.config = config - self.input_data = input_dataset - self.kernels = kernels - - # def load_fieldset(self): - # """Load fieldset for simulation.""" - # # paths = self.input_data.get_fieldset_paths() - # ... - - def get_output_path(self, output_dir: Path) -> Path: - """Get output path for results.""" - return output_dir / f"{self.name}.zarr" - - def run(self): - """Run instrument simulation.""" - with yaspin( - text=f"Simulating {self.name} measurements... ", - side="right", - spinner=ship_spinner, - ) as spinner: - self.simulate() - spinner.ok("✅") - - @abc.abstractmethod - def simulate(self): - """Simulate instrument measurements.""" - ... diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index 3c7bc0ee5..fc4c13621 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -5,7 +5,7 @@ import numpy as np from parcels import FieldSet, ParticleSet, ScipyParticle, Variable -from virtualship.instruments.master import InputDataset +from virtualship.instruments.base import InputDataset from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/types.py b/src/virtualship/instruments/types.py new file mode 100644 index 000000000..9ae221e9a --- /dev/null +++ b/src/virtualship/instruments/types.py @@ -0,0 +1,18 @@ +from enum import Enum + + +class InstrumentType(Enum): + """Types of the instruments.""" + + CTD = "CTD" + CTD_BGC = "CTD_BGC" + DRIFTER = "DRIFTER" + ARGO_FLOAT = "ARGO_FLOAT" + XBT = "XBT" + ADCP = "ADCP" + UNDERWATER_ST = "UNDERWATER_ST" + + @property + def is_underway(self) -> bool: + """Return True if instrument is an underway instrument (ADCP, UNDERWATER_ST).""" + return self in {InstrumentType.ADCP, InstrumentType.UNDERWATER_ST} diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index 4351d08e9..5d3b52ef7 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -6,7 +6,7 @@ import numpy as np from parcels import FieldSet, JITParticle, ParticleSet, Variable -from virtualship.instruments.master import InputDataset +from virtualship.instruments.base import InputDataset from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index 1b501119d..d7559fd0b 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -9,6 +9,7 @@ import yaml from virtualship.errors import InstrumentsConfigError, ScheduleError +from virtualship.instruments.types import InstrumentType from virtualship.utils import _validate_numeric_mins_to_timedelta from .location import Location @@ -17,7 +18,6 @@ if TYPE_CHECKING: from parcels import FieldSet from virtualship.expedition.input_data import InputData - from virtualship.instruments.master import InstrumentType projection: pyproj.Geod = pyproj.Geod(ellps="WGS84") @@ -44,10 +44,8 @@ def from_yaml(cls, file_path: str) -> Expedition: data = yaml.safe_load(file) return Expedition(**data) - def get_instruments(self): + def get_instruments(self) -> set[InstrumentType]: """Return a set of unique InstrumentType enums used in the expedition.""" - from virtualship.instruments.master import InstrumentType - instruments_in_expedition = [] # from waypoints for waypoint in self.schedule.waypoints: @@ -115,8 +113,6 @@ def verify( """ print("\nVerifying route... ") - from virtualship.instruments.master import InstrumentType - if check_space_time_region and self.space_time_region is None: raise ScheduleError( "space_time_region not found in schedule, please define it to fetch the data." @@ -183,7 +179,6 @@ def verify( # check that ship will arrive on time at each waypoint (in case no unexpected event happen) time = self.waypoints[0].time - for wp_i, (wp, wp_next) in enumerate( zip(self.waypoints, self.waypoints[1:], strict=False) ): @@ -418,8 +413,6 @@ def verify(self, expedition: Expedition) -> None: Removes instrument configs not present in the schedule and checks that all scheduled instruments are configured. Raises ConfigError if any scheduled instrument is missing a config. """ - from virtualship.instruments.master import InstrumentType - instruments_in_expedition = expedition.get_instruments() instrument_config_map = { InstrumentType.ARGO_FLOAT: "argo_float_config", From fdc0e6e4f23908cd954cf06f79dbbfd17261bcfe Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 27 Oct 2025 21:18:14 +0100 Subject: [PATCH 26/97] implement instrument registration and input dataset retrieval --- src/virtualship/cli/_fetch.py | 23 ++----------------- src/virtualship/instruments/adcp.py | 3 +++ src/virtualship/instruments/argo_float.py | 3 +++ src/virtualship/instruments/ctd.py | 3 +++ src/virtualship/instruments/ctd_bgc.py | 3 +++ src/virtualship/instruments/drifter.py | 3 +++ src/virtualship/instruments/master.py | 0 .../instruments/ship_underwater_st.py | 3 +++ src/virtualship/instruments/xbt.py | 3 +++ src/virtualship/utils.py | 15 ++++++++++++ 10 files changed, 38 insertions(+), 21 deletions(-) delete mode 100644 src/virtualship/instruments/master.py diff --git a/src/virtualship/cli/_fetch.py b/src/virtualship/cli/_fetch.py index 17fb403ab..50dda3a52 100644 --- a/src/virtualship/cli/_fetch.py +++ b/src/virtualship/cli/_fetch.py @@ -16,6 +16,7 @@ _dump_yaml, _generic_load_yaml, _get_expedition, + get_input_dataset_class, ) if TYPE_CHECKING: @@ -107,29 +108,9 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None coordinates_selection_method="outside", ) - # Direct mapping from InstrumentType to input dataset class - from virtualship.instruments.adcp import ADCPInputDataset - from virtualship.instruments.argo_float import ArgoFloatInputDataset - from virtualship.instruments.ctd import CTDInputDataset - from virtualship.instruments.ctd_bgc import CTD_BGCInputDataset - from virtualship.instruments.drifter import DrifterInputDataset - from virtualship.instruments.ship_underwater_st import Underwater_STInputDataset - from virtualship.instruments.types import InstrumentType - from virtualship.instruments.xbt import XBTInputDataset - - INSTRUMENT_INPUT_DATASET_MAP = { - InstrumentType.CTD: CTDInputDataset, - InstrumentType.CTD_BGC: CTD_BGCInputDataset, - InstrumentType.DRIFTER: DrifterInputDataset, - InstrumentType.ARGO_FLOAT: ArgoFloatInputDataset, - InstrumentType.XBT: XBTInputDataset, - InstrumentType.ADCP: ADCPInputDataset, - InstrumentType.UNDERWATER_ST: Underwater_STInputDataset, - } - # Only keep instruments present in the expedition for itype in instruments_in_expedition: - input_dataset_class = INSTRUMENT_INPUT_DATASET_MAP.get(itype) + input_dataset_class = get_input_dataset_class(itype) if input_dataset_class is None: continue click.echo( diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index 078effebd..1bc67e004 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -6,7 +6,9 @@ from parcels import FieldSet, ParticleSet, ScipyParticle, Variable from virtualship.instruments.base import InputDataset +from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime +from virtualship.utils import register_instrument @dataclass @@ -32,6 +34,7 @@ def _sample_velocity(particle, fieldset, time): ) +@register_instrument(InstrumentType.ADCP) class ADCPInputDataset(InputDataset): """Input dataset for ADCP instrument.""" diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index c654fd1e1..7f7d23a16 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -15,7 +15,9 @@ Variable, ) from virtualship.instruments.base import InputDataset +from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime +from virtualship.utils import register_instrument @dataclass @@ -116,6 +118,7 @@ def _check_error(particle, fieldset, time): particle.delete() +@register_instrument(InstrumentType.ARGO_FLOAT) class ArgoFloatInputDataset(InputDataset): """Input dataset for ArgoFloat instrument.""" diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 7d5846011..cd8fd330d 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -7,7 +7,9 @@ from parcels import FieldSet, JITParticle, ParticleSet, Variable from virtualship.instruments.base import InputDataset +from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime +from virtualship.utils import register_instrument @dataclass @@ -54,6 +56,7 @@ def _ctd_cast(particle, fieldset, time): particle.delete() +@register_instrument(InstrumentType.CTD) class CTDInputDataset(InputDataset): """Input dataset for CTD instrument.""" diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 1f45ce95f..92f717dbf 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -7,7 +7,9 @@ from parcels import FieldSet, JITParticle, ParticleSet, Variable from virtualship.instruments.base import InputDataset +from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime +from virtualship.utils import register_instrument @dataclass @@ -79,6 +81,7 @@ def _ctd_bgc_cast(particle, fieldset, time): particle.delete() +@register_instrument(InstrumentType.CTD_BGC) class CTD_BGCInputDataset(InputDataset): """Input dataset object for CTD_BGC instrument.""" diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index df765a0de..4ca0d087a 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -7,7 +7,9 @@ from parcels import AdvectionRK4, FieldSet, JITParticle, ParticleSet, Variable from virtualship.instruments.base import InputDataset +from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime +from virtualship.utils import register_instrument @dataclass @@ -41,6 +43,7 @@ def _check_lifetime(particle, fieldset, time): particle.delete() +@register_instrument(InstrumentType.DRIFTER) class DrifterInputDataset(InputDataset): """Input dataset for Drifter instrument.""" diff --git a/src/virtualship/instruments/master.py b/src/virtualship/instruments/master.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index fc4c13621..accfb5b38 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -6,7 +6,9 @@ from parcels import FieldSet, ParticleSet, ScipyParticle, Variable from virtualship.instruments.base import InputDataset +from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime +from virtualship.utils import register_instrument @dataclass @@ -34,6 +36,7 @@ def _sample_temperature(particle, fieldset, time): particle.T = fieldset.T[time, particle.depth, particle.lat, particle.lon] +@register_instrument(InstrumentType.UNDERWATER_ST) class Underwater_STInputDataset(InputDataset): """Input dataset for Underwater_ST instrument.""" diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index 5d3b52ef7..f5ec5fd08 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -7,7 +7,9 @@ from parcels import FieldSet, JITParticle, ParticleSet, Variable from virtualship.instruments.base import InputDataset +from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime +from virtualship.utils import register_instrument @dataclass @@ -55,6 +57,7 @@ def _xbt_cast(particle, fieldset, time): particle_ddepth = particle.max_depth - particle.depth +@register_instrument(InstrumentType.XBT) class XBTInputDataset(InputDataset): """Input dataset for XBT instrument.""" diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index 0a39d0356..e1054da1a 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -239,3 +239,18 @@ def _get_expedition(expedition_dir: Path) -> Expedition: "🚢 ", ], ) + +# InstrumentType -> InputDataset registry and registration utilities. +INSTRUMENT_INPUT_DATASET_MAP = {} + + +def register_instrument(instrument_type): + def decorator(cls): + INSTRUMENT_INPUT_DATASET_MAP[instrument_type] = cls + return cls + + return decorator + + +def get_input_dataset_class(instrument_type): + return INSTRUMENT_INPUT_DATASET_MAP.get(instrument_type) From 95e6cbd0d47797f69dee2745094812e288666dd5 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 27 Oct 2025 21:44:24 +0100 Subject: [PATCH 27/97] refactor: reorganize imports in instrument test files for consistency --- src/virtualship/instruments/__init__.py | 20 +++++++++++++++++++ .../{test_master.py => test_base.py} | 16 ++++++--------- tests/instruments/test_ctd.py | 2 +- 3 files changed, 27 insertions(+), 11 deletions(-) rename tests/instruments/{test_master.py => test_base.py} (86%) diff --git a/src/virtualship/instruments/__init__.py b/src/virtualship/instruments/__init__.py index 5324da2cd..b593ed38b 100644 --- a/src/virtualship/instruments/__init__.py +++ b/src/virtualship/instruments/__init__.py @@ -1 +1,21 @@ """Instruments in VirtualShip.""" + +from . import ( + adcp, + argo_float, + ctd, + ctd_bgc, + drifter, + ship_underwater_st, + xbt, +) + +__all__ = [ + "adcp", + "argo_float", + "ctd", + "ctd_bgc", + "drifter", + "ship_underwater_st", + "xbt", +] diff --git a/tests/instruments/test_master.py b/tests/instruments/test_base.py similarity index 86% rename from tests/instruments/test_master.py rename to tests/instruments/test_base.py index f84238d09..f41920924 100644 --- a/tests/instruments/test_master.py +++ b/tests/instruments/test_base.py @@ -3,16 +3,14 @@ import pytest -from virtualship.instruments.master import ( - InputDataset, - InstrumentType, - get_instruments_registry, -) +from virtualship.instruments.base import InputDataset +from virtualship.instruments.types import InstrumentType from virtualship.models.space_time_region import ( SpaceTimeRegion, SpatialRange, TimeRange, ) +from virtualship.utils import get_input_dataset_class class DummyInputDataset(InputDataset): @@ -85,7 +83,7 @@ def test_dummyinputdataset_initialization(dummy_space_time_region): assert ds.credentials["username"] == "u" -@patch("virtualship.models.instruments.copernicusmarine.subset") +@patch("virtualship.instruments.base.copernicusmarine.subset") def test_download_data_calls_subset(mock_subset, dummy_space_time_region): ds = DummyInputDataset( name="test", @@ -102,8 +100,6 @@ def test_download_data_calls_subset(mock_subset, dummy_space_time_region): def test_all_instruments_have_input_class(): - registry = get_instruments_registry() for instrument in InstrumentType: - entry = registry.get(instrument) - assert entry is not None, f"No registry entry for {instrument}" - assert entry.get("input_class") is not None, f"No input_class for {instrument}" + input_class = get_input_dataset_class(instrument) + assert input_class is not None, f"No input_class for {instrument}" diff --git a/tests/instruments/test_ctd.py b/tests/instruments/test_ctd.py index 14e0a2765..0a8edcfa0 100644 --- a/tests/instruments/test_ctd.py +++ b/tests/instruments/test_ctd.py @@ -9,8 +9,8 @@ import numpy as np import xarray as xr -from parcels import Field, FieldSet +from parcels import Field, FieldSet from virtualship.instruments.ctd import CTD, simulate_ctd from virtualship.models import Location, Spacetime From 01cee18ff6e52781ad9335cbbe4de512b265cca7 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 28 Oct 2025 11:50:21 +0100 Subject: [PATCH 28/97] further refactoring: instrument classes to use a unified InputDataset and Instrument structure --- src/virtualship/cli/_fetch.py | 2 +- src/virtualship/expedition/do_expedition.py | 81 +++--- src/virtualship/expedition/input_data.py | 255 ------------------ src/virtualship/instruments/adcp.py | 138 +++++----- src/virtualship/instruments/argo_float.py | 177 ++++++------ src/virtualship/instruments/base.py | 94 +++++-- src/virtualship/instruments/ctd.py | 193 +++++++------ src/virtualship/instruments/ctd_bgc.py | 231 ++++++++-------- src/virtualship/instruments/drifter.py | 180 ++++++------- .../instruments/ship_underwater_st.py | 124 ++++----- src/virtualship/instruments/xbt.py | 197 +++++++------- src/virtualship/models/expedition.py | 25 +- src/virtualship/utils.py | 22 +- 13 files changed, 746 insertions(+), 973 deletions(-) delete mode 100644 src/virtualship/expedition/input_data.py diff --git a/src/virtualship/cli/_fetch.py b/src/virtualship/cli/_fetch.py index 50dda3a52..67695695e 100644 --- a/src/virtualship/cli/_fetch.py +++ b/src/virtualship/cli/_fetch.py @@ -112,7 +112,7 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None for itype in instruments_in_expedition: input_dataset_class = get_input_dataset_class(itype) if input_dataset_class is None: - continue + raise RuntimeError(f"No input dataset class found for type {itype}.") click.echo( f"\n\n{(' Fetching data for: ' + itype.value + ' ').center(80, '=')}\n\n" ) diff --git a/src/virtualship/expedition/do_expedition.py b/src/virtualship/expedition/do_expedition.py index 921ea528d..ef5b6037e 100644 --- a/src/virtualship/expedition/do_expedition.py +++ b/src/virtualship/expedition/do_expedition.py @@ -6,17 +6,11 @@ import pyproj -from virtualship.cli._fetch import get_existing_download, get_space_time_region_hash -from virtualship.models import Expedition, Schedule -from virtualship.utils import ( - CHECKPOINT, - _get_expedition, -) +from virtualship.models import Schedule +from virtualship.utils import CHECKPOINT, _get_expedition, get_instrument_class from .checkpoint import Checkpoint from .expedition_cost import expedition_cost -from .input_data import InputData -from .simulate_measurements import simulate_measurements from .simulate_schedule import ScheduleProblem, simulate_schedule # projection used to sail between waypoints @@ -51,6 +45,7 @@ def do_expedition(expedition_dir: str | Path, input_data: Path | None = None) -> checkpoint.verify(expedition.schedule) # load fieldsets + _load_input_data = [] # TEMPORARY! loaded_input_data = _load_input_data( expedition_dir=expedition_dir, expedition=expedition, @@ -106,56 +101,42 @@ def do_expedition(expedition_dir: str | Path, input_data: Path | None = None) -> # simulate measurements print("\nSimulating measurements. This may take a while...\n") - simulate_measurements( - expedition_dir, - expedition.instruments_config, - loaded_input_data, - schedule_results.measurements_to_simulate, - ) - print("\nAll measurement simulations are complete.") - print("\n----- EXPEDITION RESULTS ------") - print("\nYour expedition has concluded successfully!") - print( - f"Your measurements can be found in the '{expedition_dir}/results' directory." - ) - print("\n------------- END -------------\n") + # TODO: this is where XYZInstrument.run() could be called instead of simulate_measurements!? + # TODO: this time maybe looping through measurements to simulate in some form... + # TODO: first in explicit per instrument, then think about whether can be automated more...not the end of the world if just have to explain in documentation that changes must be made here... + instruments_in_expedition = expedition.get_instruments() -def _load_input_data( - expedition_dir: Path, - expedition: Expedition, - input_data: Path | None, -) -> InputData: - """ - Load the input data. + for itype in instruments_in_expedition: + instrument_class = get_instrument_class(itype) + if instrument_class is None: + raise RuntimeError(f"No instrument class found for type {itype}.") - :param expedition_dir: Directory of the expedition. - :param expedition: Expedition object. - :param input_data: Folder containing input data. - :return: InputData object. - """ - if input_data is None: - space_time_region_hash = get_space_time_region_hash( - expedition.schedule.space_time_region + measurements = schedule_results.measurements_to_simulate.get(itype.name.lower()) + + instrument_class.run( + expedition_dir.joinpath("results", f"{itype.name.lower()}.zarr"), + measurements=measurements, + fieldset=loaded_input_data.get_fieldset_for_instrument_type(itype), + expedition=expedition, ) - input_data = get_existing_download(expedition_dir, space_time_region_hash) - assert input_data is not None, ( - "Input data hasn't been found. Have you run the `virtualship fetch` command?" - ) + # simulate_measurements( + # expedition_dir, + # expedition.instruments_config, + # loaded_input_data, + # schedule_results.measurements_to_simulate, + # ) - return InputData.load( - directory=input_data, - load_adcp=expedition.instruments_config.adcp_config is not None, - load_argo_float=expedition.instruments_config.argo_float_config is not None, - load_ctd=expedition.instruments_config.ctd_config is not None, - load_ctd_bgc=expedition.instruments_config.ctd_bgc_config is not None, - load_drifter=expedition.instruments_config.drifter_config is not None, - load_xbt=expedition.instruments_config.xbt_config is not None, - load_ship_underwater_st=expedition.instruments_config.ship_underwater_st_config - is not None, + print("\nAll measurement simulations are complete.") + + print("\n----- EXPEDITION RESULTS ------") + print("\nYour expedition has concluded successfully!") + print( + f"Your measurements can be found in the '{expedition_dir}/results' directory." ) + print("\n------------- END -------------\n") def _load_checkpoint(expedition_dir: Path) -> Checkpoint | None: diff --git a/src/virtualship/expedition/input_data.py b/src/virtualship/expedition/input_data.py deleted file mode 100644 index fa48e0a70..000000000 --- a/src/virtualship/expedition/input_data.py +++ /dev/null @@ -1,255 +0,0 @@ -"""InputData class.""" - -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import Path - -from parcels import Field, FieldSet - - -@dataclass -class InputData: - """A collection of fieldsets that function as input data for simulation.""" - - adcp_fieldset: FieldSet | None - argo_float_fieldset: FieldSet | None - ctd_fieldset: FieldSet | None - ctd_bgc_fieldset: FieldSet | None - drifter_fieldset: FieldSet | None - xbt_fieldset: FieldSet | None - ship_underwater_st_fieldset: FieldSet | None - - @classmethod - def load( - cls, - directory: str | Path, - load_adcp: bool, - load_argo_float: bool, - load_ctd: bool, - load_ctd_bgc: bool, - load_drifter: bool, - load_xbt: bool, - load_ship_underwater_st: bool, - ) -> InputData: - """ - Create an instance of this class from netCDF files. - - For now this function makes a lot of assumption about file location and contents. - - :param directory: Input data directory. - :param load_adcp: Whether to load the ADCP fieldset. - :param load_argo_float: Whether to load the argo float fieldset. - :param load_ctd: Whether to load the CTD fieldset. - :param load_ctd_bgc: Whether to load the CTD BGC fieldset. - :param load_drifter: Whether to load the drifter fieldset. - :param load_ship_underwater_st: Whether to load the ship underwater ST fieldset. - :returns: An instance of this class with loaded fieldsets. - """ - directory = Path(directory) - if load_drifter: - drifter_fieldset = cls._load_drifter_fieldset(directory) - else: - drifter_fieldset = None - if load_argo_float: - argo_float_fieldset = cls._load_argo_float_fieldset(directory) - else: - argo_float_fieldset = None - if load_ctd_bgc: - ctd_bgc_fieldset = cls._load_ctd_bgc_fieldset(directory) - else: - ctd_bgc_fieldset = None - if load_adcp or load_ctd or load_ship_underwater_st or load_xbt: - ship_fieldset = cls._load_ship_fieldset(directory) - if load_adcp: - adcp_fieldset = ship_fieldset - else: - adcp_fieldset = None - if load_ctd: - ctd_fieldset = ship_fieldset - else: - ctd_fieldset = None - if load_ship_underwater_st: - ship_underwater_st_fieldset = ship_fieldset - else: - ship_underwater_st_fieldset = None - if load_xbt: - xbt_fieldset = ship_fieldset - else: - xbt_fieldset = None - - return InputData( - adcp_fieldset=adcp_fieldset, - argo_float_fieldset=argo_float_fieldset, - ctd_fieldset=ctd_fieldset, - ctd_bgc_fieldset=ctd_bgc_fieldset, - drifter_fieldset=drifter_fieldset, - xbt_fieldset=xbt_fieldset, - ship_underwater_st_fieldset=ship_underwater_st_fieldset, - ) - - @classmethod - def _load_ship_fieldset(cls, directory: Path) -> FieldSet: - filenames = { - "U": directory.joinpath("ship_uv.nc"), - "V": directory.joinpath("ship_uv.nc"), - "S": directory.joinpath("ship_s.nc"), - "T": directory.joinpath("ship_t.nc"), - } - variables = {"U": "uo", "V": "vo", "S": "so", "T": "thetao"} - dimensions = { - "lon": "longitude", - "lat": "latitude", - "time": "time", - "depth": "depth", - } - - # create the fieldset and set interpolation methods - fieldset = FieldSet.from_netcdf( - filenames, variables, dimensions, allow_time_extrapolation=True - ) - fieldset.T.interp_method = "linear_invdist_land_tracer" - fieldset.S.interp_method = "linear_invdist_land_tracer" - - # make depth negative - for g in fieldset.gridset.grids: - g.negate_depth() - - # add bathymetry data - bathymetry_file = directory.joinpath("bathymetry.nc") - bathymetry_variables = ("bathymetry", "deptho") - bathymetry_dimensions = {"lon": "longitude", "lat": "latitude"} - bathymetry_field = Field.from_netcdf( - bathymetry_file, bathymetry_variables, bathymetry_dimensions - ) - # make depth negative - bathymetry_field.data = -bathymetry_field.data - fieldset.add_field(bathymetry_field) - - # read in data already - fieldset.computeTimeChunk(0, 1) - - return fieldset - - @classmethod - def _load_ctd_bgc_fieldset(cls, directory: Path) -> FieldSet: - filenames = { - "U": directory.joinpath("ship_uv.nc"), - "V": directory.joinpath("ship_uv.nc"), - "o2": directory.joinpath("ctd_bgc_o2.nc"), - "chl": directory.joinpath("ctd_bgc_chl.nc"), - "no3": directory.joinpath("ctd_bgc_no3.nc"), - "po4": directory.joinpath("ctd_bgc_po4.nc"), - "ph": directory.joinpath("ctd_bgc_ph.nc"), - "phyc": directory.joinpath("ctd_bgc_phyc.nc"), - "nppv": directory.joinpath("ctd_bgc_nppv.nc"), - } - variables = { - "U": "uo", - "V": "vo", - "o2": "o2", - "chl": "chl", - "no3": "no3", - "po4": "po4", - "ph": "ph", - "phyc": "phyc", - "nppv": "nppv", - } - dimensions = { - "lon": "longitude", - "lat": "latitude", - "time": "time", - "depth": "depth", - } - - fieldset = FieldSet.from_netcdf( - filenames, variables, dimensions, allow_time_extrapolation=True - ) - fieldset.o2.interp_method = "linear_invdist_land_tracer" - fieldset.chl.interp_method = "linear_invdist_land_tracer" - fieldset.no3.interp_method = "linear_invdist_land_tracer" - fieldset.po4.interp_method = "linear_invdist_land_tracer" - fieldset.ph.interp_method = "linear_invdist_land_tracer" - fieldset.phyc.interp_method = "linear_invdist_land_tracer" - fieldset.nppv.interp_method = "linear_invdist_land_tracer" - - # make depth negative - for g in fieldset.gridset.grids: - g.negate_depth() - - # add bathymetry data - bathymetry_file = directory.joinpath("bathymetry.nc") - bathymetry_variables = ("bathymetry", "deptho") - bathymetry_dimensions = {"lon": "longitude", "lat": "latitude"} - bathymetry_field = Field.from_netcdf( - bathymetry_file, bathymetry_variables, bathymetry_dimensions - ) - # make depth negative - bathymetry_field.data = -bathymetry_field.data - fieldset.add_field(bathymetry_field) - - # read in data already - fieldset.computeTimeChunk(0, 1) - - return fieldset - - @classmethod - def _load_drifter_fieldset(cls, directory: Path) -> FieldSet: - filenames = { - "U": directory.joinpath("drifter_uv.nc"), - "V": directory.joinpath("drifter_uv.nc"), - "T": directory.joinpath("drifter_t.nc"), - } - variables = {"U": "uo", "V": "vo", "T": "thetao"} - dimensions = { - "lon": "longitude", - "lat": "latitude", - "time": "time", - "depth": "depth", - } - - fieldset = FieldSet.from_netcdf( - filenames, variables, dimensions, allow_time_extrapolation=False - ) - fieldset.T.interp_method = "linear_invdist_land_tracer" - - # make depth negative - for g in fieldset.gridset.grids: - g.negate_depth() - - # read in data already - fieldset.computeTimeChunk(0, 1) - - return fieldset - - @classmethod - def _load_argo_float_fieldset(cls, directory: Path) -> FieldSet: - filenames = { - "U": directory.joinpath("argo_float_uv.nc"), - "V": directory.joinpath("argo_float_uv.nc"), - "S": directory.joinpath("argo_float_s.nc"), - "T": directory.joinpath("argo_float_t.nc"), - } - variables = {"U": "uo", "V": "vo", "S": "so", "T": "thetao"} - dimensions = { - "lon": "longitude", - "lat": "latitude", - "time": "time", - "depth": "depth", - } - - fieldset = FieldSet.from_netcdf( - filenames, variables, dimensions, allow_time_extrapolation=False - ) - fieldset.T.interp_method = "linear_invdist_land_tracer" - fieldset.S.interp_method = "linear_invdist_land_tracer" - - # make depth negative - for g in fieldset.gridset.grids: - if max(g.depth) > 0: - g.negate_depth() - - # read in data already - fieldset.computeTimeChunk(0, 1) - - return fieldset diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index 1bc67e004..5caab5bfc 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -4,11 +4,11 @@ import numpy as np -from parcels import FieldSet, ParticleSet, ScipyParticle, Variable -from virtualship.instruments.base import InputDataset +from parcels import ParticleSet, ScipyParticle, Variable +from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime -from virtualship.utils import register_instrument +from virtualship.utils import register_input_dataset, register_instrument @dataclass @@ -34,7 +34,7 @@ def _sample_velocity(particle, fieldset, time): ) -@register_instrument(InstrumentType.ADCP) +@register_input_dataset(InstrumentType.ADCP) class ADCPInputDataset(InputDataset): """Input dataset for ADCP instrument.""" @@ -69,78 +69,64 @@ def get_datasets_dict(self) -> dict: } -# TODO: uncomment when ready for new simulation logic! -# class ADCPInstrument(instruments.Instrument): -# """ADCP instrument class.""" - -# def __init__( -# self, -# config, -# input_dataset, -# kernels, -# ): -# """Initialise with instrument's name.""" -# super().__init__(ADCP.name, config, input_dataset, kernels) - -# def simulate(self): -# """Simulate measurements.""" -# ... - - -# TODO: to be replaced with new simulation logic -## -- old simulation code - - -def simulate_adcp( - fieldset: FieldSet, - out_path: str | Path, - max_depth: float, - min_depth: float, - num_bins: int, - sample_points: list[Spacetime], -) -> None: - """ - Use Parcels to simulate an ADCP in a fieldset. - - :param fieldset: The fieldset to simulate the ADCP in. - :param out_path: The path to write the results to. - :param max_depth: Maximum depth the ADCP can measure. - :param min_depth: Minimum depth the ADCP can measure. - :param num_bins: How many samples to take in the complete range between max_depth and min_depth. - :param sample_points: The places and times to sample at. - """ - sample_points.sort(key=lambda p: p.time) - - bins = np.linspace(max_depth, min_depth, num_bins) - num_particles = len(bins) - particleset = ParticleSet.from_list( - fieldset=fieldset, - pclass=_ADCPParticle, - lon=np.full( - num_particles, 0.0 - ), # initial lat/lon are irrelevant and will be overruled later. - lat=np.full(num_particles, 0.0), - depth=bins, - time=0, # same for time - ) - - # define output file for the simulation - # outputdt set to infinite as we just want to write at the end of every call to 'execute' - out_file = particleset.ParticleFile(name=out_path, outputdt=np.inf) - - for point in sample_points: - particleset.lon_nextloop[:] = point.location.lon - particleset.lat_nextloop[:] = point.location.lat - particleset.time_nextloop[:] = fieldset.time_origin.reltime( - np.datetime64(point.time) +@register_instrument(InstrumentType.ADCP) +class ADCPInstrument(Instrument): + """ADCP instrument class.""" + + def __init__( + self, + input_dataset: InputDataset, + ): + """Initialize ADCPInstrument.""" + filenames = { + "UV": input_dataset.data_dir.joinpath(f"{input_dataset.name}_uv.nc"), + } + variables = {"UV": ["uo", "vo"]} + super().__init__( + input_dataset, + filenames, + variables, + add_bathymetry=False, + allow_time_extrapolation=True, ) - # perform one step using the particleset - # dt and runtime are set so exactly one step is made. - particleset.execute( - [_sample_velocity], - dt=1, - runtime=1, - verbose_progress=False, - output_file=out_file, + def simulate( + self, + out_path: str | Path, + max_depth: float, + min_depth: float, + num_bins: int, + sample_points: list[Spacetime], + ) -> None: + """Simulate ADCP measurements.""" + sample_points.sort(key=lambda p: p.time) + + fieldset = self.load_input_data() + + bins = np.linspace(max_depth, min_depth, num_bins) + num_particles = len(bins) + particleset = ParticleSet.from_list( + fieldset=fieldset, + pclass=_ADCPParticle, + lon=np.full(num_particles, 0.0), + lat=np.full(num_particles, 0.0), + depth=bins, + time=0, ) + + out_file = particleset.ParticleFile(name=out_path, outputdt=np.inf) + + for point in sample_points: + particleset.lon_nextloop[:] = point.location.lon + particleset.lat_nextloop[:] = point.location.lat + particleset.time_nextloop[:] = fieldset.time_origin.reltime( + np.datetime64(point.time) + ) + + particleset.execute( + [_sample_velocity], + dt=1, + runtime=1, + verbose_progress=False, + output_file=out_file, + ) diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index 7f7d23a16..66b25bdf3 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -8,16 +8,15 @@ from parcels import ( AdvectionRK4, - FieldSet, JITParticle, ParticleSet, StatusCode, Variable, ) -from virtualship.instruments.base import InputDataset +from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime -from virtualship.utils import register_instrument +from virtualship.utils import register_input_dataset, register_instrument @dataclass @@ -118,7 +117,7 @@ def _check_error(particle, fieldset, time): particle.delete() -@register_instrument(InstrumentType.ARGO_FLOAT) +@register_input_dataset(InstrumentType.ARGO_FLOAT) class ArgoFloatInputDataset(InputDataset): """Input dataset for ArgoFloat instrument.""" @@ -163,89 +162,89 @@ def get_datasets_dict(self) -> dict: } -# class ArgoFloatInstrument(instruments.Instrument): -# """ArgoFloat instrument class.""" - -# def __init__( -# self, -# config, -# input_dataset, -# kernels, -# ): -# """Initialise with instrument's name.""" -# super().__init__(ArgoFloat.name, config, input_dataset, kernels) - -# def simulate(self): -# """Simulate measurements.""" -# ... - - -def simulate_argo_floats( - fieldset: FieldSet, - out_path: str | Path, - argo_floats: list[ArgoFloat], - outputdt: timedelta, - endtime: datetime | None, -) -> None: - """ - Use Parcels to simulate a set of Argo floats in a fieldset. - - :param fieldset: The fieldset to simulate the Argo floats in. - :param out_path: The path to write the results to. - :param argo_floats: A list of Argo floats to simulate. - :param outputdt: Interval which dictates the update frequency of file output during simulation - :param endtime: Stop at this time, or if None, continue until the end of the fieldset. - """ - DT = 10.0 # dt of Argo float simulation integrator - - if len(argo_floats) == 0: - print( - "No Argo floats provided. Parcels currently crashes when providing an empty particle set, so no argo floats simulation will be done and no files will be created." +@register_instrument(InstrumentType.ARGO_FLOAT) +class ArgoFloatInstrument(Instrument): + """ArgoFloat instrument class.""" + + def __init__( + self, + input_dataset: InputDataset, + ): + """Initialize ArgoFloatInstrument.""" + filenames = { + "UV": input_dataset.data_dir.joinpath("argo_float_uv.nc"), + "S": input_dataset.data_dir.joinpath("argo_float_s.nc"), + "T": input_dataset.data_dir.joinpath("argo_float_t.nc"), + } + variables = {"UV": ["uo", "vo"], "S": "so", "T": "thetao"} + super().__init__( + input_dataset, + filenames, + variables, + add_bathymetry=False, + allow_time_extrapolation=False, + ) + + def simulate( + self, + argo_floats: list[ArgoFloat], + out_path: str | Path, + outputdt: timedelta, + endtime: datetime | None = None, + ) -> None: + """Simulate Argo float measurements.""" + DT = 10.0 # dt of Argo float simulation integrator + + if len(argo_floats) == 0: + print( + "No Argo floats provided. Parcels currently crashes when providing an empty particle set, so no argo floats simulation will be done and no files will be created." + ) + # TODO when Parcels supports it this check can be removed. + return + + fieldset = self.load_input_data() + + # define parcel particles + argo_float_particleset = ParticleSet( + fieldset=fieldset, + pclass=_ArgoParticle, + lat=[argo.spacetime.location.lat for argo in argo_floats], + lon=[argo.spacetime.location.lon for argo in argo_floats], + depth=[argo.min_depth for argo in argo_floats], + time=[argo.spacetime.time for argo in argo_floats], + min_depth=[argo.min_depth for argo in argo_floats], + max_depth=[argo.max_depth for argo in argo_floats], + drift_depth=[argo.drift_depth for argo in argo_floats], + vertical_speed=[argo.vertical_speed for argo in argo_floats], + cycle_days=[argo.cycle_days for argo in argo_floats], + drift_days=[argo.drift_days for argo in argo_floats], + ) + + # define output file for the simulation + out_file = argo_float_particleset.ParticleFile( + name=out_path, outputdt=outputdt, chunks=[len(argo_float_particleset), 100] + ) + + # get earliest between fieldset end time and provide end time + fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) + if endtime is None: + actual_endtime = fieldset_endtime + elif endtime > fieldset_endtime: + print("WARN: Requested end time later than fieldset end time.") + actual_endtime = fieldset_endtime + else: + actual_endtime = np.timedelta64(endtime) + + # execute simulation + argo_float_particleset.execute( + [ + _argo_float_vertical_movement, + AdvectionRK4, + _keep_at_surface, + _check_error, + ], + endtime=actual_endtime, + dt=DT, + output_file=out_file, + verbose_progress=True, ) - # TODO when Parcels supports it this check can be removed. - return - - # define parcel particles - argo_float_particleset = ParticleSet( - fieldset=fieldset, - pclass=_ArgoParticle, - lat=[argo.spacetime.location.lat for argo in argo_floats], - lon=[argo.spacetime.location.lon for argo in argo_floats], - depth=[argo.min_depth for argo in argo_floats], - time=[argo.spacetime.time for argo in argo_floats], - min_depth=[argo.min_depth for argo in argo_floats], - max_depth=[argo.max_depth for argo in argo_floats], - drift_depth=[argo.drift_depth for argo in argo_floats], - vertical_speed=[argo.vertical_speed for argo in argo_floats], - cycle_days=[argo.cycle_days for argo in argo_floats], - drift_days=[argo.drift_days for argo in argo_floats], - ) - - # define output file for the simulation - out_file = argo_float_particleset.ParticleFile( - name=out_path, outputdt=outputdt, chunks=[len(argo_float_particleset), 100] - ) - - # get earliest between fieldset end time and provide end time - fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) - if endtime is None: - actual_endtime = fieldset_endtime - elif endtime > fieldset_endtime: - print("WARN: Requested end time later than fieldset end time.") - actual_endtime = fieldset_endtime - else: - actual_endtime = np.timedelta64(endtime) - - # execute simulation - argo_float_particleset.execute( - [ - _argo_float_vertical_movement, - AdvectionRK4, - _keep_at_surface, - _check_error, - ], - endtime=actual_endtime, - dt=DT, - output_file=out_file, - verbose_progress=True, - ) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index dfdb9c6ab..2fab47959 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -1,11 +1,10 @@ import abc -from collections.abc import Callable from datetime import timedelta -from pathlib import Path import copernicusmarine import yaspin +from parcels import Field, FieldSet from virtualship.models import SpaceTimeRegion from virtualship.utils import ship_spinner @@ -37,7 +36,6 @@ def __init__( @abc.abstractmethod def get_datasets_dict(self) -> dict: """Get parameters for instrument's variable(s) specific data download.""" - ... def download_data(self) -> None: """Download data for the instrument using copernicusmarine.""" @@ -69,41 +67,89 @@ def download_data(self) -> None: class Instrument(abc.ABC): - """Base class for instruments.""" + """Base class for instruments and their simulation.""" def __init__( self, - name: str, - config, input_dataset: InputDataset, - kernels: list[Callable], + filenames: dict, + variables: dict, + add_bathymetry: bool, + allow_time_extrapolation: bool, + bathymetry_file: str = "bathymetry.nc", ): """Initialise instrument.""" - self.name = name - self.config = config self.input_data = input_dataset - self.kernels = kernels + self.name = input_dataset.name + self.directory = input_dataset.data_dir + self.filenames = filenames + self.variables = variables + self.dimensions = { + "lon": "longitude", + "lat": "latitude", + "time": "time", + "depth": "depth", + } # same dimensions for all instruments + self.bathymetry_file = self.directory.joinpath(bathymetry_file) + self.add_bathymetry = add_bathymetry + self.allow_time_extrapolation = allow_time_extrapolation + + def load_input_data(self) -> FieldSet: + """Load and return the input data as a FieldSet for the instrument.""" + # TODO: this should mean can delete input_data.py! + + # TODO: hopefully simulate_measurements can also be removed! And maybe the list of e.g. ctds ('measurements') to run can be added to higher level like do_expedition.py...? I think as they already do... + + # TODO: can simulate_schedule.py be refactored to be contained in base.py and repsective instrument files too...? - # def load_fieldset(self): - # """Load fieldset for simulation.""" - # # paths = self.input_data.get_fieldset_paths() - # ... + # TODO: what do I need to do about automatic registration of Instrument classes...? - def get_output_path(self, output_dir: Path) -> Path: - """Get output path for results.""" - return output_dir / f"{self.name}.zarr" + # TODO: tests will need updating...! - def run(self): + # TODO: think about combining InputDataset and Instrument classes together! + + try: + fieldset = FieldSet.from_netcdf( + self.filenames, + self.variables, + self.dimensions, + allow_time_extrapolation=self.allow_time_extrapolation, + ) + except FileNotFoundError as e: + raise FileNotFoundError( + f"Input data for instrument {self.name} not found. Have you run the `virtualship fetch` command??" + ) from e + + # interpolation methods + for var in self.variables: + getattr(fieldset, var).interp_method = "linear_invdist_land_tracer" + # depth negative + for g in fieldset.gridset.grids: + g.negate_depth() + # bathymetry data + if self.add_bathymetry: + bathymetry_field = Field.from_netcdf( + self.bathymetry_file, + self.bathymetry_variables, + self.bathymetry_dimensions, + ) + bathymetry_field.data = -bathymetry_field.data + fieldset.add_field(bathymetry_field) + fieldset.computeTimeChunk(0, 1) # read in data already + return fieldset + + @abc.abstractmethod + def simulate(self): + """Simulate instrument measurements.""" + + def run(self, *args, **kwargs): """Run instrument simulation.""" + # TODO: this will have to be able to handle the non-spinner/instead progress bar for drifters and argos! + with yaspin( text=f"Simulating {self.name} measurements... ", side="right", spinner=ship_spinner, ) as spinner: - self.simulate() + self.simulate(*args, **kwargs) spinner.ok("✅") - - @abc.abstractmethod - def simulate(self): - """Simulate instrument measurements.""" - ... diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index cd8fd330d..15e810416 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -5,11 +5,11 @@ import numpy as np -from parcels import FieldSet, JITParticle, ParticleSet, Variable -from virtualship.instruments.base import InputDataset +from parcels import JITParticle, ParticleSet, Variable +from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType -from virtualship.models.spacetime import Spacetime -from virtualship.utils import register_instrument +from virtualship.models import Spacetime +from virtualship.utils import register_input_dataset @dataclass @@ -56,7 +56,7 @@ def _ctd_cast(particle, fieldset, time): particle.delete() -@register_instrument(InstrumentType.CTD) +@register_input_dataset(InstrumentType.CTD) class CTDInputDataset(InputDataset): """Input dataset for CTD instrument.""" @@ -94,102 +94,101 @@ def get_datasets_dict(self) -> dict: } -# class CTDInstrument(instruments.Instrument): -# """CTD instrument class.""" - -# def __init__( -# self, -# config, -# input_dataset, -# kernels, -# ): -# """Initialise with instrument's name.""" -# super().__init__(CTD.name, config, input_dataset, kernels) - -# def simulate(self): -# """Simulate measurements.""" -# ... - - -def simulate_ctd( - fieldset: FieldSet, - out_path: str | Path, - ctds: list[CTD], - outputdt: timedelta, -) -> None: - """ - Use Parcels to simulate a set of CTDs in a fieldset. - - :param fieldset: The fieldset to simulate the CTDs in. - :param out_path: The path to write the results to. - :param ctds: A list of CTDs to simulate. - :param outputdt: Interval which dictates the update frequency of file output during simulation - :raises ValueError: Whenever provided CTDs, fieldset, are not compatible with this function. - """ - WINCH_SPEED = 1.0 # sink and rise speed in m/s - DT = 10.0 # dt of CTD simulation integrator - - if len(ctds) == 0: - print( - "No CTDs provided. Parcels currently crashes when providing an empty particle set, so no CTD simulation will be done and no files will be created." - ) - # TODO when Parcels supports it this check can be removed. - return - - fieldset_starttime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[0]) - fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) +class CTDInstrument(Instrument): + """CTD instrument class.""" - # deploy time for all ctds should be later than fieldset start time - if not all( - [np.datetime64(ctd.spacetime.time) >= fieldset_starttime for ctd in ctds] + def __init__( + self, + input_dataset: InputDataset, ): - raise ValueError("CTD deployed before fieldset starts.") - - # depth the ctd will go to. shallowest between ctd max depth and bathymetry. - max_depths = [ - max( - ctd.max_depth, - fieldset.bathymetry.eval( - z=0, y=ctd.spacetime.location.lat, x=ctd.spacetime.location.lon, time=0 - ), + """Initialize CTDInstrument.""" + filenames = { + "S": input_dataset.data_dir.joinpath(f"{input_dataset.name}_s.nc"), + "T": input_dataset.data_dir.joinpath(f"{input_dataset.name}_t.nc"), + } + variables = {"S": "so", "T": "thetao"} + + super().__init__( + input_dataset, + filenames, + variables, + add_bathymetry=True, + allow_time_extrapolation=True, ) - for ctd in ctds - ] - # CTD depth can not be too shallow, because kernel would break. - # This shallow is not useful anyway, no need to support. - if not all([max_depth <= -DT * WINCH_SPEED for max_depth in max_depths]): - raise ValueError( - f"CTD max_depth or bathymetry shallower than maximum {-DT * WINCH_SPEED}" + def simulate( + self, ctds: list[CTD], out_path: str | Path, outputdt: timedelta + ) -> None: + """Simulate CTD measurements.""" + WINCH_SPEED = 1.0 # sink and rise speed in m/s + DT = 10.0 # dt of CTD simulation integrator + + if len(ctds) == 0: + print( + "No CTDs provided. Parcels currently crashes when providing an empty particle set, so no CTD simulation will be done and no files will be created." + ) + # TODO when Parcels supports it this check can be removed. + return + + fieldset = self.load_input_data() + + fieldset_starttime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[0]) + fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) + + # deploy time for all ctds should be later than fieldset start time + if not all( + [np.datetime64(ctd.spacetime.time) >= fieldset_starttime for ctd in ctds] + ): + raise ValueError("CTD deployed before fieldset starts.") + + # depth the ctd will go to. shallowest between ctd max depth and bathymetry. + max_depths = [ + max( + ctd.max_depth, + fieldset.bathymetry.eval( + z=0, + y=ctd.spacetime.location.lat, + x=ctd.spacetime.location.lon, + time=0, + ), + ) + for ctd in ctds + ] + + # CTD depth can not be too shallow, because kernel would break. + # This shallow is not useful anyway, no need to support. + if not all([max_depth <= -DT * WINCH_SPEED for max_depth in max_depths]): + raise ValueError( + f"CTD max_depth or bathymetry shallower than maximum {-DT * WINCH_SPEED}" + ) + + # define parcel particles + ctd_particleset = ParticleSet( + fieldset=fieldset, + pclass=_CTDParticle, + lon=[ctd.spacetime.location.lon for ctd in ctds], + lat=[ctd.spacetime.location.lat for ctd in ctds], + depth=[ctd.min_depth for ctd in ctds], + time=[ctd.spacetime.time for ctd in ctds], + max_depth=max_depths, + min_depth=[ctd.min_depth for ctd in ctds], + winch_speed=[WINCH_SPEED for _ in ctds], ) - # define parcel particles - ctd_particleset = ParticleSet( - fieldset=fieldset, - pclass=_CTDParticle, - lon=[ctd.spacetime.location.lon for ctd in ctds], - lat=[ctd.spacetime.location.lat for ctd in ctds], - depth=[ctd.min_depth for ctd in ctds], - time=[ctd.spacetime.time for ctd in ctds], - max_depth=max_depths, - min_depth=[ctd.min_depth for ctd in ctds], - winch_speed=[WINCH_SPEED for _ in ctds], - ) - - # define output file for the simulation - out_file = ctd_particleset.ParticleFile(name=out_path, outputdt=outputdt) - - # execute simulation - ctd_particleset.execute( - [_sample_salinity, _sample_temperature, _ctd_cast], - endtime=fieldset_endtime, - dt=DT, - verbose_progress=False, - output_file=out_file, - ) - - # there should be no particles left, as they delete themselves when they resurface - if len(ctd_particleset.particledata) != 0: - raise ValueError( - "Simulation ended before CTD resurfaced. This most likely means the field time dimension did not match the simulation time span." + # define output file for the simulation + out_file = ctd_particleset.ParticleFile(name=out_path, outputdt=outputdt) + + # execute simulation + ctd_particleset.execute( + [_sample_salinity, _sample_temperature, _ctd_cast], + endtime=fieldset_endtime, + dt=DT, + verbose_progress=False, + output_file=out_file, ) + + # there should be no particles left, as they delete themselves when they resurface + if len(ctd_particleset.particledata) != 0: + raise ValueError( + "Simulation ended before CTD resurfaced. This most likely means the field time dimension did not match the simulation time span." + ) diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 92f717dbf..85bf02f5e 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -5,11 +5,11 @@ import numpy as np -from parcels import FieldSet, JITParticle, ParticleSet, Variable -from virtualship.instruments.base import InputDataset +from parcels import JITParticle, ParticleSet, Variable +from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime -from virtualship.utils import register_instrument +from virtualship.utils import register_input_dataset, register_instrument @dataclass @@ -81,7 +81,7 @@ def _ctd_bgc_cast(particle, fieldset, time): particle.delete() -@register_instrument(InstrumentType.CTD_BGC) +@register_input_dataset(InstrumentType.CTD_BGC) class CTD_BGCInputDataset(InputDataset): """Input dataset object for CTD_BGC instrument.""" @@ -149,117 +149,128 @@ def get_datasets_dict(self) -> dict: } -# class CTD_BGCInstrument(instruments.Instrument): -# """CTD_BGC instrument class.""" - -# def __init__( -# self, -# config, -# input_dataset, -# kernels, -# ): -# """Initialise with instrument's name.""" -# super().__init__(CTD_BGC.name, config, input_dataset, kernels) - -# def simulate(self): -# """Simulate measurements.""" -# ... - - -def simulate_ctd_bgc( - fieldset: FieldSet, - out_path: str | Path, - ctd_bgcs: list[CTD_BGC], - outputdt: timedelta, -) -> None: - """ - Use Parcels to simulate a set of BGC CTDs in a fieldset. - - :param fieldset: The fieldset to simulate the BGC CTDs in. - :param out_path: The path to write the results to. - :param ctds: A list of BGC CTDs to simulate. - :param outputdt: Interval which dictates the update frequency of file output during simulation - :raises ValueError: Whenever provided BGC CTDs, fieldset, are not compatible with this function. - """ - WINCH_SPEED = 1.0 # sink and rise speed in m/s - DT = 10.0 # dt of CTD simulation integrator - - if len(ctd_bgcs) == 0: - print( - "No BGC CTDs provided. Parcels currently crashes when providing an empty particle set, so no BGC CTD simulation will be done and no files will be created." - ) - # TODO when Parcels supports it this check can be removed. - return +@register_instrument(InstrumentType.CTD_BGC) +class CTD_BGCInstrument(Instrument): + """CTD_BGC instrument class.""" - fieldset_starttime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[0]) - fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) + def __init__( + self, + input_dataset: InputDataset, + ): + """Initialize CTD_BGCInstrument.""" + filenames = { + "o2": input_dataset.data_dir.joinpath("ctd_bgc_o2.nc"), + "chl": input_dataset.data_dir.joinpath("ctd_bgc_chl.nc"), + "no3": input_dataset.data_dir.joinpath("ctd_bgc_no3.nc"), + "po4": input_dataset.data_dir.joinpath("ctd_bgc_po4.nc"), + "ph": input_dataset.data_dir.joinpath("ctd_bgc_ph.nc"), + "phyc": input_dataset.data_dir.joinpath("ctd_bgc_phyc.nc"), + "zooc": input_dataset.data_dir.joinpath("ctd_bgc_zooc.nc"), + "nppv": input_dataset.data_dir.joinpath("ctd_bgc_nppv.nc"), + } + variables = { + "o2": "o2", + "chl": "chl", + "no3": "no3", + "po4": "po4", + "ph": "ph", + "phyc": "phyc", + "zooc": "zooc", + "nppv": "nppv", + } + super().__init__( + input_dataset, + filenames, + variables, + add_bathymetry=True, + allow_time_extrapolation=True, + ) - # deploy time for all ctds should be later than fieldset start time - if not all( - [ - np.datetime64(ctd_bgc.spacetime.time) >= fieldset_starttime + def simulate( + self, ctd_bgcs: list[CTD_BGC], out_path: str | Path, outputdt: timedelta + ) -> None: + """Simulate BGC CTD measurements using Parcels.""" + WINCH_SPEED = 1.0 # sink and rise speed in m/s + DT = 10.0 # dt of CTD simulation integrator + + if len(ctd_bgcs) == 0: + print( + "No BGC CTDs provided. Parcels currently crashes when providing an empty particle set, so no BGC CTD simulation will be done and no files will be created." + ) + # TODO when Parcels supports it this check can be removed. + return + + fieldset = self.load_input_data() + + fieldset_starttime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[0]) + fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) + + # deploy time for all ctds should be later than fieldset start time + if not all( + [ + np.datetime64(ctd_bgc.spacetime.time) >= fieldset_starttime + for ctd_bgc in ctd_bgcs + ] + ): + raise ValueError("BGC CTD deployed before fieldset starts.") + + # depth the bgc ctd will go to. shallowest between bgc ctd max depth and bathymetry. + max_depths = [ + max( + ctd_bgc.max_depth, + fieldset.bathymetry.eval( + z=0, + y=ctd_bgc.spacetime.location.lat, + x=ctd_bgc.spacetime.location.lon, + time=0, + ), + ) for ctd_bgc in ctd_bgcs ] - ): - raise ValueError("BGC CTD deployed before fieldset starts.") - - # depth the bgc ctd will go to. shallowest between bgc ctd max depth and bathymetry. - max_depths = [ - max( - ctd_bgc.max_depth, - fieldset.bathymetry.eval( - z=0, - y=ctd_bgc.spacetime.location.lat, - x=ctd_bgc.spacetime.location.lon, - time=0, - ), - ) - for ctd_bgc in ctd_bgcs - ] - # CTD depth can not be too shallow, because kernel would break. - # This shallow is not useful anyway, no need to support. - if not all([max_depth <= -DT * WINCH_SPEED for max_depth in max_depths]): - raise ValueError( - f"BGC CTD max_depth or bathymetry shallower than maximum {-DT * WINCH_SPEED}" + # CTD depth can not be too shallow, because kernel would break. + # This shallow is not useful anyway, no need to support. + if not all([max_depth <= -DT * WINCH_SPEED for max_depth in max_depths]): + raise ValueError( + f"BGC CTD max_depth or bathymetry shallower than maximum {-DT * WINCH_SPEED}" + ) + + # define parcel particles + ctd_bgc_particleset = ParticleSet( + fieldset=fieldset, + pclass=_CTD_BGCParticle, + lon=[ctd_bgc.spacetime.location.lon for ctd_bgc in ctd_bgcs], + lat=[ctd_bgc.spacetime.location.lat for ctd_bgc in ctd_bgcs], + depth=[ctd_bgc.min_depth for ctd_bgc in ctd_bgcs], + time=[ctd_bgc.spacetime.time for ctd_bgc in ctd_bgcs], + max_depth=max_depths, + min_depth=[ctd_bgc.min_depth for ctd_bgc in ctd_bgcs], + winch_speed=[WINCH_SPEED for _ in ctd_bgcs], ) - # define parcel particles - ctd_bgc_particleset = ParticleSet( - fieldset=fieldset, - pclass=_CTD_BGCParticle, - lon=[ctd_bgc.spacetime.location.lon for ctd_bgc in ctd_bgcs], - lat=[ctd_bgc.spacetime.location.lat for ctd_bgc in ctd_bgcs], - depth=[ctd_bgc.min_depth for ctd_bgc in ctd_bgcs], - time=[ctd_bgc.spacetime.time for ctd_bgc in ctd_bgcs], - max_depth=max_depths, - min_depth=[ctd_bgc.min_depth for ctd_bgc in ctd_bgcs], - winch_speed=[WINCH_SPEED for _ in ctd_bgcs], - ) - - # define output file for the simulation - out_file = ctd_bgc_particleset.ParticleFile(name=out_path, outputdt=outputdt) - - # execute simulation - ctd_bgc_particleset.execute( - [ - _sample_o2, - _sample_chlorophyll, - _sample_nitrate, - _sample_phosphate, - _sample_ph, - _sample_phytoplankton, - _sample_primary_production, - _ctd_bgc_cast, - ], - endtime=fieldset_endtime, - dt=DT, - verbose_progress=False, - output_file=out_file, - ) - - # there should be no particles left, as they delete themselves when they resurface - if len(ctd_bgc_particleset.particledata) != 0: - raise ValueError( - "Simulation ended before BGC CTD resurfaced. This most likely means the field time dimension did not match the simulation time span." + # define output file for the simulation + out_file = ctd_bgc_particleset.ParticleFile(name=out_path, outputdt=outputdt) + + # execute simulation + ctd_bgc_particleset.execute( + [ + _sample_o2, + _sample_chlorophyll, + _sample_nitrate, + _sample_phosphate, + _sample_ph, + _sample_phytoplankton, + _sample_primary_production, + _ctd_bgc_cast, + ], + endtime=fieldset_endtime, + dt=DT, + verbose_progress=False, + output_file=out_file, ) + + # there should be no particles left, as they delete themselves when they resurface + if len(ctd_bgc_particleset.particledata) != 0: + raise ValueError( + "Simulation ended before BGC CTD resurfaced. This most likely means the field time dimension did not match the simulation time span." + ) diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index 4ca0d087a..186383ce8 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -5,11 +5,11 @@ import numpy as np -from parcels import AdvectionRK4, FieldSet, JITParticle, ParticleSet, Variable -from virtualship.instruments.base import InputDataset +from parcels import AdvectionRK4, JITParticle, ParticleSet, Variable +from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime -from virtualship.utils import register_instrument +from virtualship.utils import register_input_dataset, register_instrument @dataclass @@ -43,7 +43,7 @@ def _check_lifetime(particle, fieldset, time): particle.delete() -@register_instrument(InstrumentType.DRIFTER) +@register_input_dataset(InstrumentType.DRIFTER) class DrifterInputDataset(InputDataset): """Input dataset for Drifter instrument.""" @@ -83,91 +83,91 @@ def get_datasets_dict(self) -> dict: } -# class DrifterInstrument(instruments.Instrument): -# """Drifter instrument class.""" - -# def __init__( -# self, -# config, -# input_dataset, -# kernels, -# ): -# """Initialise with instrument's name.""" -# super().__init__(Drifter.name, config, input_dataset, kernels) - -# def simulate(self): -# """Simulate measurements.""" -# ... - - -def simulate_drifters( - fieldset: FieldSet, - out_path: str | Path, - drifters: list[Drifter], - outputdt: timedelta, - dt: timedelta, - endtime: datetime | None = None, -) -> None: - """ - Use Parcels to simulate a set of drifters in a fieldset. - - :param fieldset: The fieldset to simulate the Drifters in. - :param out_path: The path to write the results to. - :param drifters: A list of drifters to simulate. - :param outputdt: Interval which dictates the update frequency of file output during simulation. - :param dt: Dt for integration. - :param endtime: Stop at this time, or if None, continue until the end of the fieldset or until all drifters ended. If this is earlier than the last drifter ended or later than the end of the fieldset, a warning will be printed. - """ - if len(drifters) == 0: - print( - "No drifters provided. Parcels currently crashes when providing an empty particle set, so no drifter simulation will be done and no files will be created." - ) - # TODO when Parcels supports it this check can be removed. - return - - # define parcel particles - drifter_particleset = ParticleSet( - fieldset=fieldset, - pclass=_DrifterParticle, - lat=[drifter.spacetime.location.lat for drifter in drifters], - lon=[drifter.spacetime.location.lon for drifter in drifters], - depth=[drifter.depth for drifter in drifters], - time=[drifter.spacetime.time for drifter in drifters], - has_lifetime=[1 if drifter.lifetime is not None else 0 for drifter in drifters], - lifetime=[ - 0 if drifter.lifetime is None else drifter.lifetime.total_seconds() - for drifter in drifters - ], - ) - - # define output file for the simulation - out_file = drifter_particleset.ParticleFile( - name=out_path, outputdt=outputdt, chunks=[len(drifter_particleset), 100] - ) - - # get earliest between fieldset end time and provide end time - fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) - if endtime is None: - actual_endtime = fieldset_endtime - elif endtime > fieldset_endtime: - print("WARN: Requested end time later than fieldset end time.") - actual_endtime = fieldset_endtime - else: - actual_endtime = np.timedelta64(endtime) - - # execute simulation - drifter_particleset.execute( - [AdvectionRK4, _sample_temperature, _check_lifetime], - endtime=actual_endtime, - dt=dt, - output_file=out_file, - verbose_progress=True, - ) - - # if there are more particles left than the number of drifters with an indefinite endtime, warn the user - if len(drifter_particleset.particledata) > len( - [d for d in drifters if d.lifetime is None] +@register_instrument(InstrumentType.DRIFTER) +class DrifterInstrument(Instrument): + """Drifter instrument class.""" + + def __init__( + self, + input_dataset: InputDataset, ): - print( - "WARN: Some drifters had a life time beyond the end time of the fieldset or the requested end time." + """Initialize DrifterInstrument.""" + filenames = { + "UV": input_dataset.data_dir.joinpath("drifter_uv.nc"), + "T": input_dataset.data_dir.joinpath("drifter_t.nc"), + } + variables = {"UV": ["uo", "vo"], "T": "thetao"} + super().__init__( + input_dataset, + filenames, + variables, + add_bathymetry=False, + allow_time_extrapolation=False, ) + + def simulate( + self, + drifters: list[Drifter], + out_path: str | Path, + outputdt: timedelta, + dt: timedelta, + endtime: datetime | None = None, + ) -> None: + """Simulate Drifter measurements.""" + if len(drifters) == 0: + print( + "No drifters provided. Parcels currently crashes when providing an empty particle set, so no drifter simulation will be done and no files will be created." + ) + # TODO when Parcels supports it this check can be removed. + return + + fieldset = self.load_input_data() + + # define parcel particles + drifter_particleset = ParticleSet( + fieldset=fieldset, + pclass=_DrifterParticle, + lat=[drifter.spacetime.location.lat for drifter in drifters], + lon=[drifter.spacetime.location.lon for drifter in drifters], + depth=[drifter.depth for drifter in drifters], + time=[drifter.spacetime.time for drifter in drifters], + has_lifetime=[ + 1 if drifter.lifetime is not None else 0 for drifter in drifters + ], + lifetime=[ + 0 if drifter.lifetime is None else drifter.lifetime.total_seconds() + for drifter in drifters + ], + ) + + # define output file for the simulation + out_file = drifter_particleset.ParticleFile( + name=out_path, outputdt=outputdt, chunks=[len(drifter_particleset), 100] + ) + + # get earliest between fieldset end time and provide end time + fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) + if endtime is None: + actual_endtime = fieldset_endtime + elif endtime > fieldset_endtime: + print("WARN: Requested end time later than fieldset end time.") + actual_endtime = fieldset_endtime + else: + actual_endtime = np.timedelta64(endtime) + + # execute simulation + drifter_particleset.execute( + [AdvectionRK4, _sample_temperature, _check_lifetime], + endtime=actual_endtime, + dt=dt, + output_file=out_file, + verbose_progress=True, + ) + + # if there are more particles left than the number of drifters with an indefinite endtime, warn the user + if len(drifter_particleset.particledata) > len( + [d for d in drifters if d.lifetime is None] + ): + print( + "WARN: Some drifters had a life time beyond the end time of the fieldset or the requested end time." + ) diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index accfb5b38..371b4485a 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -4,11 +4,11 @@ import numpy as np -from parcels import FieldSet, ParticleSet, ScipyParticle, Variable -from virtualship.instruments.base import InputDataset +from parcels import ParticleSet, ScipyParticle, Variable +from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime -from virtualship.utils import register_instrument +from virtualship.utils import register_input_dataset, register_instrument @dataclass @@ -36,7 +36,7 @@ def _sample_temperature(particle, fieldset, time): particle.T = fieldset.T[time, particle.depth, particle.lat, particle.lon] -@register_instrument(InstrumentType.UNDERWATER_ST) +@register_input_dataset(InstrumentType.UNDERWATER_ST) class Underwater_STInputDataset(InputDataset): """Input dataset for Underwater_ST instrument.""" @@ -76,67 +76,61 @@ def get_datasets_dict(self) -> dict: } -# class Underwater_STInstrument(instruments.Instrument): -# """Underwater_ST instrument class.""" - -# def __init__( -# self, -# config, -# input_dataset, -# kernels, -# ): -# """Initialise with instrument's name.""" -# super().__init__(Underwater_ST.name, config, input_dataset, kernels) - -# def simulate(self): -# """Simulate measurements.""" -# ... - - -def simulate_ship_underwater_st( - fieldset: FieldSet, - out_path: str | Path, - depth: float, - sample_points: list[Spacetime], -) -> None: - """ - Use Parcels to simulate underway data, measuring salinity and temperature at the given depth along the ship track in a fieldset. - - :param fieldset: The fieldset to simulate the sampling in. - :param out_path: The path to write the results to. - :param depth: The depth at which to measure. 0 is water surface, negative is into the water. - :param sample_points: The places and times to sample at. - """ - sample_points.sort(key=lambda p: p.time) - - particleset = ParticleSet.from_list( - fieldset=fieldset, - pclass=_ShipSTParticle, - lon=0.0, # initial lat/lon are irrelevant and will be overruled later - lat=0.0, - depth=depth, - time=0, # same for time - ) - - # define output file for the simulation - # outputdt set to infinie as we want to just want to write at the end of every call to 'execute' - out_file = particleset.ParticleFile(name=out_path, outputdt=np.inf) - - # iterate over each point, manually set lat lon time, then - # execute the particle set for one step, performing one set of measurement - for point in sample_points: - particleset.lon_nextloop[:] = point.location.lon - particleset.lat_nextloop[:] = point.location.lat - particleset.time_nextloop[:] = fieldset.time_origin.reltime( - np.datetime64(point.time) +@register_instrument(InstrumentType.UNDERWATER_ST) +class Underwater_STInstrument(Instrument): + """Underwater_ST instrument class.""" + + def __init__( + self, + input_dataset: InputDataset, + ): + """Initialize Underwater_STInstrument.""" + filenames = { + "S": input_dataset.data_dir.joinpath(f"{input_dataset.name}_s.nc"), + "T": input_dataset.data_dir.joinpath(f"{input_dataset.name}_t.nc"), + } + variables = {"S": "so", "T": "thetao"} + super().__init__( + input_dataset, + filenames, + variables, + add_bathymetry=False, + allow_time_extrapolation=True, ) - # perform one step using the particleset - # dt and runtime are set so exactly one step is made. - particleset.execute( - [_sample_salinity, _sample_temperature], - dt=1, - runtime=1, - verbose_progress=False, - output_file=out_file, + def simulate( + self, + out_path: str | Path, + depth: float, + sample_points: list[Spacetime], + ) -> None: + """Simulate underway salinity and temperature measurements.""" + sample_points.sort(key=lambda p: p.time) + + fieldset = self.load_input_data() + + particleset = ParticleSet.from_list( + fieldset=fieldset, + pclass=_ShipSTParticle, + lon=0.0, + lat=0.0, + depth=depth, + time=0, ) + + out_file = particleset.ParticleFile(name=out_path, outputdt=np.inf) + + for point in sample_points: + particleset.lon_nextloop[:] = point.location.lon + particleset.lat_nextloop[:] = point.location.lat + particleset.time_nextloop[:] = fieldset.time_origin.reltime( + np.datetime64(point.time) + ) + + particleset.execute( + [_sample_salinity, _sample_temperature], + dt=1, + runtime=1, + verbose_progress=False, + output_file=out_file, + ) diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index f5ec5fd08..c2fec98d3 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -5,11 +5,11 @@ import numpy as np -from parcels import FieldSet, JITParticle, ParticleSet, Variable -from virtualship.instruments.base import InputDataset +from parcels import JITParticle, ParticleSet, Variable +from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime -from virtualship.utils import register_instrument +from virtualship.utils import register_input_dataset, register_instrument @dataclass @@ -57,7 +57,7 @@ def _xbt_cast(particle, fieldset, time): particle_ddepth = particle.max_depth - particle.depth -@register_instrument(InstrumentType.XBT) +@register_input_dataset(InstrumentType.XBT) class XBTInputDataset(InputDataset): """Input dataset for XBT instrument.""" @@ -102,106 +102,105 @@ def get_datasets_dict(self) -> dict: } -# class XBTInstrument(instruments.Instrument): -# """XBT instrument class.""" - -# def __init__( -# self, -# config, -# input_dataset, -# kernels, -# ): -# """Initialise with instrument's name.""" -# super().__init__(XBT.name, config, input_dataset, kernels) - -# def simulate(self): -# """Simulate measurements.""" -# ... - - -def simulate_xbt( - fieldset: FieldSet, - out_path: str | Path, - xbts: list[XBT], - outputdt: timedelta, -) -> None: - """ - Use Parcels to simulate a set of XBTs in a fieldset. - - :param fieldset: The fieldset to simulate the XBTs in. - :param out_path: The path to write the results to. - :param xbts: A list of XBTs to simulate. - :param outputdt: Interval which dictates the update frequency of file output during simulation - :raises ValueError: Whenever provided XBTs, fieldset, are not compatible with this function. - """ - DT = 10.0 # dt of XBT simulation integrator - - if len(xbts) == 0: - print( - "No XBTs provided. Parcels currently crashes when providing an empty particle set, so no XBT simulation will be done and no files will be created." - ) - # TODO when Parcels supports it this check can be removed. - return - - fieldset_starttime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[0]) - fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) +@register_instrument(InstrumentType.XBT) +class XBTInstrument(Instrument): + """XBT instrument class.""" - # deploy time for all xbts should be later than fieldset start time - if not all( - [np.datetime64(xbt.spacetime.time) >= fieldset_starttime for xbt in xbts] + def __init__( + self, + input_dataset: InputDataset, ): - raise ValueError("XBT deployed before fieldset starts.") - - # depth the xbt will go to. shallowest between xbt max depth and bathymetry. - max_depths = [ - max( - xbt.max_depth, - fieldset.bathymetry.eval( - z=0, y=xbt.spacetime.location.lat, x=xbt.spacetime.location.lon, time=0 - ), + """Initialize XBTInstrument.""" + filenames = { + "UV": input_dataset.data_dir.joinpath("ship_uv.nc"), + "S": input_dataset.data_dir.joinpath("ship_s.nc"), + "T": input_dataset.data_dir.joinpath("ship_t.nc"), + } + variables = {"UV": ["uo", "vo"], "S": "so", "T": "thetao"} + super().__init__( + input_dataset, + filenames, + variables, + add_bathymetry=True, + allow_time_extrapolation=True, ) - for xbt in xbts - ] - # initial fall speeds - initial_fall_speeds = [xbt.fall_speed for xbt in xbts] - - # XBT depth can not be too shallow, because kernel would break. - # This shallow is not useful anyway, no need to support. - # TODO: should this be more informative? Is "maximum" right? Should tell user can't use XBT here? - for max_depth, fall_speed in zip(max_depths, initial_fall_speeds, strict=False): - if not max_depth <= -DT * fall_speed: - raise ValueError( - f"XBT max_depth or bathymetry shallower than maximum {-DT * fall_speed}" + def simulate( + self, + xbts: list[XBT], + out_path: str | Path, + outputdt: timedelta, + ) -> None: + """Simulate XBT measurements.""" + DT = 10.0 # dt of XBT simulation integrator + + if len(xbts) == 0: + print( + "No XBTs provided. Parcels currently crashes when providing an empty particle set, so no XBT simulation will be done and no files will be created." ) + # TODO when Parcels supports it this check can be removed. + return + + fieldset = self.load_input_data() + + fieldset_starttime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[0]) + fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) + + # deploy time for all xbts should be later than fieldset start time + if not all( + [np.datetime64(xbt.spacetime.time) >= fieldset_starttime for xbt in xbts] + ): + raise ValueError("XBT deployed before fieldset starts.") + + # depth the xbt will go to. shallowest between xbt max depth and bathymetry. + max_depths = [ + max( + xbt.max_depth, + fieldset.bathymetry.eval( + z=0, + y=xbt.spacetime.location.lat, + x=xbt.spacetime.location.lon, + time=0, + ), + ) + for xbt in xbts + ] + + # initial fall speeds + initial_fall_speeds = [xbt.fall_speed for xbt in xbts] + + # XBT depth can not be too shallow, because kernel would break. + for max_depth, fall_speed in zip(max_depths, initial_fall_speeds, strict=False): + if not max_depth <= -DT * fall_speed: + raise ValueError( + f"XBT max_depth or bathymetry shallower than maximum {-DT * fall_speed}" + ) + + # define xbt particles + xbt_particleset = ParticleSet( + fieldset=fieldset, + pclass=_XBTParticle, + lon=[xbt.spacetime.location.lon for xbt in xbts], + lat=[xbt.spacetime.location.lat for xbt in xbts], + depth=[xbt.min_depth for xbt in xbts], + time=[xbt.spacetime.time for xbt in xbts], + max_depth=max_depths, + min_depth=[xbt.min_depth for xbt in xbts], + fall_speed=[xbt.fall_speed for xbt in xbts], + ) - # define xbt particles - xbt_particleset = ParticleSet( - fieldset=fieldset, - pclass=_XBTParticle, - lon=[xbt.spacetime.location.lon for xbt in xbts], - lat=[xbt.spacetime.location.lat for xbt in xbts], - depth=[xbt.min_depth for xbt in xbts], - time=[xbt.spacetime.time for xbt in xbts], - max_depth=max_depths, - min_depth=[xbt.min_depth for xbt in xbts], - fall_speed=[xbt.fall_speed for xbt in xbts], - ) - - # define output file for the simulation - out_file = xbt_particleset.ParticleFile(name=out_path, outputdt=outputdt) + out_file = xbt_particleset.ParticleFile(name=out_path, outputdt=outputdt) - # execute simulation - xbt_particleset.execute( - [_sample_temperature, _xbt_cast], - endtime=fieldset_endtime, - dt=DT, - verbose_progress=False, - output_file=out_file, - ) - - # there should be no particles left, as they delete themselves when they finish profiling - if len(xbt_particleset.particledata) != 0: - raise ValueError( - "Simulation ended before XBT finished profiling. This most likely means the field time dimension did not match the simulation time span." + xbt_particleset.execute( + [_sample_temperature, _xbt_cast], + endtime=fieldset_endtime, + dt=DT, + verbose_progress=False, + output_file=out_file, ) + + # there should be no particles left, as they delete themselves when they finish profiling + if len(xbt_particleset.particledata) != 0: + raise ValueError( + "Simulation ended before XBT finished profiling. This most likely means the field time dimension did not match the simulation time span." + ) diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index d7559fd0b..9add2c1bd 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -17,7 +17,6 @@ if TYPE_CHECKING: from parcels import FieldSet - from virtualship.expedition.input_data import InputData projection: pyproj.Geod = pyproj.Geod(ellps="WGS84") @@ -88,7 +87,6 @@ class Schedule(pydantic.BaseModel): def verify( self, ship_speed: float, - input_data: InputData | None, *, check_space_time_region: bool = False, ignore_missing_fieldsets: bool = False, @@ -139,19 +137,20 @@ def verify( # check if all waypoints are in water # this is done by picking an arbitrary provided fieldset and checking if UV is not zero + # TODO: this may need to be done with generic bathymetry data, now that removed InputData! # get all available fieldsets available_fieldsets = [] - if input_data is not None: - fieldsets = [ - input_data.adcp_fieldset, - input_data.argo_float_fieldset, - input_data.ctd_fieldset, - input_data.drifter_fieldset, - input_data.ship_underwater_st_fieldset, - ] - for fs in fieldsets: - if fs is not None: - available_fieldsets.append(fs) + # if input_data is not None: + # fieldsets = [ + # input_data.adcp_fieldset, + # input_data.argo_float_fieldset, + # input_data.ctd_fieldset, + # input_data.drifter_fieldset, + # input_data.ship_underwater_st_fieldset, + # ] + # for fs in fieldsets: + # if fs is not None: + # available_fieldsets.append(fs) # check if there are any fieldsets, else it's an error if len(available_fieldsets) == 0: diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index e1054da1a..6ffaefe73 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -240,17 +240,31 @@ def _get_expedition(expedition_dir: Path) -> Expedition: ], ) -# InstrumentType -> InputDataset registry and registration utilities. -INSTRUMENT_INPUT_DATASET_MAP = {} + +# InstrumentType -> InputDataset and Instrument registry and registration utilities. +INPUT_DATASET_MAP = {} +INSTRUMENT_CLASS_MAP = {} + + +def register_input_dataset(instrument_type): + def decorator(cls): + INPUT_DATASET_MAP[instrument_type] = cls + return cls + + return decorator def register_instrument(instrument_type): def decorator(cls): - INSTRUMENT_INPUT_DATASET_MAP[instrument_type] = cls + INSTRUMENT_CLASS_MAP[instrument_type] = cls return cls return decorator def get_input_dataset_class(instrument_type): - return INSTRUMENT_INPUT_DATASET_MAP.get(instrument_type) + return INPUT_DATASET_MAP.get(instrument_type) + + +def get_instrument_class(instrument_type): + return INSTRUMENT_CLASS_MAP.get(instrument_type) From e96aa8a3775eed9cc45fea5553a7a66fa461aeaf Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 28 Oct 2025 13:45:19 +0100 Subject: [PATCH 29/97] evaporate simulate_measurements.py; centralise run logic --- src/virtualship/expedition/do_expedition.py | 57 +++--- .../expedition/simulate_measurements.py | 162 ------------------ src/virtualship/instruments/adcp.py | 39 ++--- src/virtualship/instruments/argo_float.py | 66 ++++--- src/virtualship/instruments/base.py | 21 ++- src/virtualship/instruments/ctd.py | 42 ++--- src/virtualship/instruments/ctd_bgc.py | 71 ++++---- src/virtualship/instruments/drifter.py | 62 ++++--- .../instruments/ship_underwater_st.py | 32 ++-- src/virtualship/instruments/xbt.py | 55 +++--- 10 files changed, 211 insertions(+), 396 deletions(-) delete mode 100644 src/virtualship/expedition/simulate_measurements.py diff --git a/src/virtualship/expedition/do_expedition.py b/src/virtualship/expedition/do_expedition.py index ef5b6037e..c4bc67837 100644 --- a/src/virtualship/expedition/do_expedition.py +++ b/src/virtualship/expedition/do_expedition.py @@ -44,17 +44,12 @@ def do_expedition(expedition_dir: str | Path, input_data: Path | None = None) -> # verify that schedule and checkpoint match checkpoint.verify(expedition.schedule) - # load fieldsets - _load_input_data = [] # TEMPORARY! - loaded_input_data = _load_input_data( - expedition_dir=expedition_dir, - expedition=expedition, - input_data=input_data, - ) - print("\n---- WAYPOINT VERIFICATION ----") # verify schedule is valid + # TODO: needs updating when .verify() updated to not need input_data + + loaded_input_data = [] # TODO: TEMPORARY! expedition.schedule.verify( expedition.ship_config.ship_speed_knots, loaded_input_data ) @@ -87,48 +82,34 @@ def do_expedition(expedition_dir: str | Path, input_data: Path | None = None) -> print("\n----- EXPEDITION SUMMARY ------") - # calculate expedition cost in US$ - assert expedition.schedule.waypoints[0].time is not None, ( - "First waypoint has no time. This should not be possible as it should have been verified before." - ) - time_past = schedule_results.time - expedition.schedule.waypoints[0].time - cost = expedition_cost(schedule_results, time_past) - with open(expedition_dir.joinpath("results", "cost.txt"), "w") as file: - file.writelines(f"cost: {cost} US$") - print(f"\nExpedition duration: {time_past}\nExpedition cost: US$ {cost:,.0f}.") + # expedition cost in US$ + _write_expedition_cost(expedition, schedule_results, expedition_dir) print("\n--- MEASUREMENT SIMULATIONS ---") # simulate measurements print("\nSimulating measurements. This may take a while...\n") - # TODO: this is where XYZInstrument.run() could be called instead of simulate_measurements!? - # TODO: this time maybe looping through measurements to simulate in some form... - # TODO: first in explicit per instrument, then think about whether can be automated more...not the end of the world if just have to explain in documentation that changes must be made here... - instruments_in_expedition = expedition.get_instruments() for itype in instruments_in_expedition: + # get instrument class instrument_class = get_instrument_class(itype) if instrument_class is None: raise RuntimeError(f"No instrument class found for type {itype}.") + # get measurements to simulate for this instrument measurements = schedule_results.measurements_to_simulate.get(itype.name.lower()) - instrument_class.run( - expedition_dir.joinpath("results", f"{itype.name.lower()}.zarr"), + # initialise instrument + instrument = instrument_class(expedition=expedition, directory=expedition_dir) + + # run simulation + instrument.run( measurements=measurements, - fieldset=loaded_input_data.get_fieldset_for_instrument_type(itype), - expedition=expedition, + out_path=expedition_dir.joinpath("results", f"{itype.name.lower()}.zarr"), ) - # simulate_measurements( - # expedition_dir, - # expedition.instruments_config, - # loaded_input_data, - # schedule_results.measurements_to_simulate, - # ) - print("\nAll measurement simulations are complete.") print("\n----- EXPEDITION RESULTS ------") @@ -150,3 +131,15 @@ def _load_checkpoint(expedition_dir: Path) -> Checkpoint | None: def _save_checkpoint(checkpoint: Checkpoint, expedition_dir: Path) -> None: file_path = expedition_dir.joinpath(CHECKPOINT) checkpoint.to_yaml(file_path) + + +def _write_expedition_cost(expedition, schedule_results, expedition_dir): + """Calculate the expedition cost, write it to a file, and print summary.""" + assert expedition.schedule.waypoints[0].time is not None, ( + "First waypoint has no time. This should not be possible as it should have been verified before." + ) + time_past = schedule_results.time - expedition.schedule.waypoints[0].time + cost = expedition_cost(schedule_results, time_past) + with open(expedition_dir.joinpath("results", "cost.txt"), "w") as file: + file.writelines(f"cost: {cost} US$") + print(f"\nExpedition duration: {time_past}\nExpedition cost: US$ {cost:,.0f}.") diff --git a/src/virtualship/expedition/simulate_measurements.py b/src/virtualship/expedition/simulate_measurements.py deleted file mode 100644 index 6cb2e4880..000000000 --- a/src/virtualship/expedition/simulate_measurements.py +++ /dev/null @@ -1,162 +0,0 @@ -"""simulate_measurements function.""" - -from __future__ import annotations - -import logging -from datetime import timedelta -from pathlib import Path -from typing import TYPE_CHECKING - -from yaspin import yaspin - -from virtualship.instruments.adcp import simulate_adcp -from virtualship.instruments.argo_float import simulate_argo_floats -from virtualship.instruments.ctd import simulate_ctd -from virtualship.instruments.ctd_bgc import simulate_ctd_bgc -from virtualship.instruments.drifter import simulate_drifters -from virtualship.instruments.ship_underwater_st import simulate_ship_underwater_st -from virtualship.instruments.xbt import simulate_xbt -from virtualship.models import InstrumentsConfig -from virtualship.utils import ship_spinner - -from .simulate_schedule import MeasurementsToSimulate - -if TYPE_CHECKING: - from .input_data import InputData - -# parcels logger (suppress INFO messages to prevent log being flooded) -external_logger = logging.getLogger("parcels.tools.loggers") -external_logger.setLevel(logging.WARNING) - - -def simulate_measurements( - expedition_dir: str | Path, - instruments_config: InstrumentsConfig, - input_data: InputData, - measurements: MeasurementsToSimulate, -) -> None: - """ - Simulate measurements using Parcels. - - Saves everything in expedition_dir/results. - - :param expedition_dir: Base directory of the expedition. - :param input_data: Input data for simulation. - :param measurements: The measurements to simulate. - :raises RuntimeError: In case fieldsets of configuration is not provided. Make sure to check this before calling this function. - """ - if isinstance(expedition_dir, str): - expedition_dir = Path(expedition_dir) - - if len(measurements.ship_underwater_sts) > 0: - if instruments_config.ship_underwater_st_config is None: - raise RuntimeError("No configuration for ship underwater ST provided.") - if input_data.ship_underwater_st_fieldset is None: - raise RuntimeError("No fieldset for ship underwater ST provided.") - with yaspin( - text="Simulating onboard temperature and salinity measurements... ", - side="right", - spinner=ship_spinner, - ) as spinner: - simulate_ship_underwater_st( - fieldset=input_data.ship_underwater_st_fieldset, - out_path=expedition_dir.joinpath("results", "ship_underwater_st.zarr"), - depth=-2, - sample_points=measurements.ship_underwater_sts, - ) - spinner.ok("✅") - - if len(measurements.adcps) > 0: - if instruments_config.adcp_config is None: - raise RuntimeError("No configuration for ADCP provided.") - if input_data.adcp_fieldset is None: - raise RuntimeError("No fieldset for ADCP provided.") - with yaspin( - text="Simulating onboard ADCP... ", side="right", spinner=ship_spinner - ) as spinner: - simulate_adcp( - fieldset=input_data.adcp_fieldset, - out_path=expedition_dir.joinpath("results", "adcp.zarr"), - max_depth=instruments_config.adcp_config.max_depth_meter, - min_depth=-5, - num_bins=instruments_config.adcp_config.num_bins, - sample_points=measurements.adcps, - ) - spinner.ok("✅") - - if len(measurements.ctds) > 0: - if instruments_config.ctd_config is None: - raise RuntimeError("No configuration for CTD provided.") - if input_data.ctd_fieldset is None: - raise RuntimeError("No fieldset for CTD provided.") - with yaspin( - text="Simulating CTD casts... ", side="right", spinner=ship_spinner - ) as spinner: - simulate_ctd( - out_path=expedition_dir.joinpath("results", "ctd.zarr"), - fieldset=input_data.ctd_fieldset, - ctds=measurements.ctds, - outputdt=timedelta(seconds=10), - ) - spinner.ok("✅") - - if len(measurements.ctd_bgcs) > 0: - if instruments_config.ctd_bgc_config is None: - raise RuntimeError("No configuration for CTD_BGC provided.") - if input_data.ctd_bgc_fieldset is None: - raise RuntimeError("No fieldset for CTD_BGC provided.") - with yaspin( - text="Simulating BGC CTD casts... ", side="right", spinner=ship_spinner - ) as spinner: - simulate_ctd_bgc( - out_path=expedition_dir.joinpath("results", "ctd_bgc.zarr"), - fieldset=input_data.ctd_bgc_fieldset, - ctd_bgcs=measurements.ctd_bgcs, - outputdt=timedelta(seconds=10), - ) - spinner.ok("✅") - - if len(measurements.xbts) > 0: - if instruments_config.xbt_config is None: - raise RuntimeError("No configuration for XBTs provided.") - if input_data.xbt_fieldset is None: - raise RuntimeError("No fieldset for XBTs provided.") - with yaspin( - text="Simulating XBTs... ", side="right", spinner=ship_spinner - ) as spinner: - simulate_xbt( - out_path=expedition_dir.joinpath("results", "xbts.zarr"), - fieldset=input_data.xbt_fieldset, - xbts=measurements.xbts, - outputdt=timedelta(seconds=1), - ) - spinner.ok("✅") - - if len(measurements.drifters) > 0: - print("Simulating drifters... ") - if instruments_config.drifter_config is None: - raise RuntimeError("No configuration for drifters provided.") - if input_data.drifter_fieldset is None: - raise RuntimeError("No fieldset for drifters provided.") - simulate_drifters( - out_path=expedition_dir.joinpath("results", "drifters.zarr"), - fieldset=input_data.drifter_fieldset, - drifters=measurements.drifters, - outputdt=timedelta(hours=5), - dt=timedelta(minutes=5), - endtime=None, - ) - - if len(measurements.argo_floats) > 0: - print("Simulating argo floats... ") - if instruments_config.argo_float_config is None: - raise RuntimeError("No configuration for argo floats provided.") - if input_data.argo_float_fieldset is None: - raise RuntimeError("No fieldset for argo floats provided.") - simulate_argo_floats( - out_path=expedition_dir.joinpath("results", "argo_floats.zarr"), - argo_floats=measurements.argo_floats, - fieldset=input_data.argo_float_fieldset, - outputdt=timedelta(minutes=5), - endtime=None, - ) diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index 5caab5bfc..917da1548 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -1,5 +1,4 @@ from dataclasses import dataclass -from pathlib import Path from typing import ClassVar import numpy as np @@ -7,8 +6,10 @@ from parcels import ParticleSet, ScipyParticle, Variable from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType -from virtualship.models.spacetime import Spacetime -from virtualship.utils import register_input_dataset, register_instrument +from virtualship.utils import ( + register_input_dataset, + register_instrument, +) @dataclass @@ -73,37 +74,33 @@ def get_datasets_dict(self) -> dict: class ADCPInstrument(Instrument): """ADCP instrument class.""" - def __init__( - self, - input_dataset: InputDataset, - ): + def __init__(self, name, expedition, directory): """Initialize ADCPInstrument.""" filenames = { - "UV": input_dataset.data_dir.joinpath(f"{input_dataset.name}_uv.nc"), + "UV": directory.joinpath(f"{name}_uv.nc"), } variables = {"UV": ["uo", "vo"]} super().__init__( - input_dataset, + ADCP.name, + expedition, + directory, filenames, variables, add_bathymetry=False, allow_time_extrapolation=True, ) - def simulate( - self, - out_path: str | Path, - max_depth: float, - min_depth: float, - num_bins: int, - sample_points: list[Spacetime], - ) -> None: + def simulate(self) -> None: """Simulate ADCP measurements.""" - sample_points.sort(key=lambda p: p.time) + MAX_DEPTH = self.expedition.instruments_config.adcp_config.max_depth_meter + MIN_DEPTH = -5.0 + NUM_BINS = self.instruments_config.adcp_config.num_bins + + self.measurements.sort(key=lambda p: p.time) fieldset = self.load_input_data() - bins = np.linspace(max_depth, min_depth, num_bins) + bins = np.linspace(MAX_DEPTH, MIN_DEPTH, NUM_BINS) num_particles = len(bins) particleset = ParticleSet.from_list( fieldset=fieldset, @@ -114,9 +111,9 @@ def simulate( time=0, ) - out_file = particleset.ParticleFile(name=out_path, outputdt=np.inf) + out_file = particleset.ParticleFile(name=self.out_path, outputdt=np.inf) - for point in sample_points: + for point in self.measurements: particleset.lon_nextloop[:] = point.location.lon particleset.lat_nextloop[:] = point.location.lat particleset.time_nextloop[:] = fieldset.time_origin.reltime( diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index 66b25bdf3..239b47de6 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -1,7 +1,6 @@ import math from dataclasses import dataclass -from datetime import datetime, timedelta -from pathlib import Path +from datetime import timedelta from typing import ClassVar import numpy as np @@ -147,17 +146,17 @@ def get_datasets_dict(self) -> dict: "UVdata": { "dataset_id": "cmems_mod_glo_phy-cur_anfc_0.083deg_PT6H-i", "variables": ["uo", "vo"], - "output_filename": "argo_float_uv.nc", + "output_filename": f"{self.name}_uv.nc", }, "Sdata": { "dataset_id": "cmems_mod_glo_phy-so_anfc_0.083deg_PT6H-i", "variables": ["so"], - "output_filename": "argo_float_s.nc", + "output_filename": f"{self.name}_s.nc", }, "Tdata": { "dataset_id": "cmems_mod_glo_phy-thetao_anfc_0.083deg_PT6H-i", "variables": ["thetao"], - "output_filename": "argo_float_t.nc", + "output_filename": f"{self.name}_t.nc", }, } @@ -166,36 +165,31 @@ def get_datasets_dict(self) -> dict: class ArgoFloatInstrument(Instrument): """ArgoFloat instrument class.""" - def __init__( - self, - input_dataset: InputDataset, - ): + def __init__(self, name, expedition, directory): """Initialize ArgoFloatInstrument.""" filenames = { - "UV": input_dataset.data_dir.joinpath("argo_float_uv.nc"), - "S": input_dataset.data_dir.joinpath("argo_float_s.nc"), - "T": input_dataset.data_dir.joinpath("argo_float_t.nc"), + "UV": directory.joinpath(f"{name}_uv.nc"), + "S": directory.joinpath(f"{name}_s.nc"), + "T": directory.joinpath(f"{name}_t.nc"), } variables = {"UV": ["uo", "vo"], "S": "so", "T": "thetao"} super().__init__( - input_dataset, + ArgoFloat.name, + expedition, + directory, filenames, variables, add_bathymetry=False, allow_time_extrapolation=False, ) - def simulate( - self, - argo_floats: list[ArgoFloat], - out_path: str | Path, - outputdt: timedelta, - endtime: datetime | None = None, - ) -> None: + def simulate(self) -> None: """Simulate Argo float measurements.""" DT = 10.0 # dt of Argo float simulation integrator + OUTPUT_DT = timedelta(minutes=5) + ENDTIME = None - if len(argo_floats) == 0: + if len(self.measurements) == 0: print( "No Argo floats provided. Parcels currently crashes when providing an empty particle set, so no argo floats simulation will be done and no files will be created." ) @@ -208,32 +202,34 @@ def simulate( argo_float_particleset = ParticleSet( fieldset=fieldset, pclass=_ArgoParticle, - lat=[argo.spacetime.location.lat for argo in argo_floats], - lon=[argo.spacetime.location.lon for argo in argo_floats], - depth=[argo.min_depth for argo in argo_floats], - time=[argo.spacetime.time for argo in argo_floats], - min_depth=[argo.min_depth for argo in argo_floats], - max_depth=[argo.max_depth for argo in argo_floats], - drift_depth=[argo.drift_depth for argo in argo_floats], - vertical_speed=[argo.vertical_speed for argo in argo_floats], - cycle_days=[argo.cycle_days for argo in argo_floats], - drift_days=[argo.drift_days for argo in argo_floats], + lat=[argo.spacetime.location.lat for argo in self.measurements], + lon=[argo.spacetime.location.lon for argo in self.measurements], + depth=[argo.min_depth for argo in self.measurements], + time=[argo.spacetime.time for argo in self.measurements], + min_depth=[argo.min_depth for argo in self.measurements], + max_depth=[argo.max_depth for argo in self.measurements], + drift_depth=[argo.drift_depth for argo in self.measurements], + vertical_speed=[argo.vertical_speed for argo in self.measurements], + cycle_days=[argo.cycle_days for argo in self.measurements], + drift_days=[argo.drift_days for argo in self.measurements], ) # define output file for the simulation out_file = argo_float_particleset.ParticleFile( - name=out_path, outputdt=outputdt, chunks=[len(argo_float_particleset), 100] + name=self.out_path, + outputdt=OUTPUT_DT, + chunks=[len(argo_float_particleset), 100], ) # get earliest between fieldset end time and provide end time fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) - if endtime is None: + if ENDTIME is None: actual_endtime = fieldset_endtime - elif endtime > fieldset_endtime: + elif ENDTIME > fieldset_endtime: print("WARN: Requested end time later than fieldset end time.") actual_endtime = fieldset_endtime else: - actual_endtime = np.timedelta64(endtime) + actual_endtime = np.timedelta64(ENDTIME) # execute simulation argo_float_particleset.execute( diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 2fab47959..13a2fb055 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -1,11 +1,12 @@ import abc from datetime import timedelta +from pathlib import Path import copernicusmarine import yaspin from parcels import Field, FieldSet -from virtualship.models import SpaceTimeRegion +from virtualship.models import Expedition, SpaceTimeRegion from virtualship.utils import ship_spinner @@ -71,7 +72,9 @@ class Instrument(abc.ABC): def __init__( self, - input_dataset: InputDataset, + name: str, + expedition: Expedition, + directory: Path | str, filenames: dict, variables: dict, add_bathymetry: bool, @@ -79,9 +82,9 @@ def __init__( bathymetry_file: str = "bathymetry.nc", ): """Initialise instrument.""" - self.input_data = input_dataset - self.name = input_dataset.name - self.directory = input_dataset.data_dir + self.name = name + self.expedition = expedition + self.directory = directory self.filenames = filenames self.variables = variables self.dimensions = { @@ -106,7 +109,7 @@ def load_input_data(self) -> FieldSet: # TODO: tests will need updating...! - # TODO: think about combining InputDataset and Instrument classes together! + # TODO: think about combining InputDataset and Instrument classes together! Or maybe not if they are better kept separate... try: fieldset = FieldSet.from_netcdf( @@ -139,10 +142,10 @@ def load_input_data(self) -> FieldSet: return fieldset @abc.abstractmethod - def simulate(self): + def simulate(self, measurements: list, out_path: str | Path): """Simulate instrument measurements.""" - def run(self, *args, **kwargs): + def run(self, measurements: list, out_path: str | Path) -> None: """Run instrument simulation.""" # TODO: this will have to be able to handle the non-spinner/instead progress bar for drifters and argos! @@ -151,5 +154,5 @@ def run(self, *args, **kwargs): side="right", spinner=ship_spinner, ) as spinner: - self.simulate(*args, **kwargs) + self.simulate(measurements, out_path) spinner.ok("✅") diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 15e810416..82cace94a 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -1,6 +1,5 @@ from dataclasses import dataclass from datetime import timedelta -from pathlib import Path from typing import ClassVar import numpy as np @@ -97,33 +96,31 @@ def get_datasets_dict(self) -> dict: class CTDInstrument(Instrument): """CTD instrument class.""" - def __init__( - self, - input_dataset: InputDataset, - ): + def __init__(self, name, expedition, directory): """Initialize CTDInstrument.""" filenames = { - "S": input_dataset.data_dir.joinpath(f"{input_dataset.name}_s.nc"), - "T": input_dataset.data_dir.joinpath(f"{input_dataset.name}_t.nc"), + "S": directory.data_dir.joinpath(f"{name}_s.nc"), + "T": directory.data_dir.joinpath(f"{name}_t.nc"), } variables = {"S": "so", "T": "thetao"} super().__init__( - input_dataset, + CTD.name, + expedition, + directory, filenames, variables, add_bathymetry=True, allow_time_extrapolation=True, ) - def simulate( - self, ctds: list[CTD], out_path: str | Path, outputdt: timedelta - ) -> None: + def simulate(self) -> None: """Simulate CTD measurements.""" WINCH_SPEED = 1.0 # sink and rise speed in m/s DT = 10.0 # dt of CTD simulation integrator + OUTPUT_DT = timedelta(seconds=10) # output dt for CTD simulation - if len(ctds) == 0: + if len(self.measurements) == 0: print( "No CTDs provided. Parcels currently crashes when providing an empty particle set, so no CTD simulation will be done and no files will be created." ) @@ -137,7 +134,10 @@ def simulate( # deploy time for all ctds should be later than fieldset start time if not all( - [np.datetime64(ctd.spacetime.time) >= fieldset_starttime for ctd in ctds] + [ + np.datetime64(ctd.spacetime.time) >= fieldset_starttime + for ctd in self.measurements + ] ): raise ValueError("CTD deployed before fieldset starts.") @@ -152,7 +152,7 @@ def simulate( time=0, ), ) - for ctd in ctds + for ctd in self.measurements ] # CTD depth can not be too shallow, because kernel would break. @@ -166,17 +166,17 @@ def simulate( ctd_particleset = ParticleSet( fieldset=fieldset, pclass=_CTDParticle, - lon=[ctd.spacetime.location.lon for ctd in ctds], - lat=[ctd.spacetime.location.lat for ctd in ctds], - depth=[ctd.min_depth for ctd in ctds], - time=[ctd.spacetime.time for ctd in ctds], + lon=[ctd.spacetime.location.lon for ctd in self.measurements], + lat=[ctd.spacetime.location.lat for ctd in self.measurements], + depth=[ctd.min_depth for ctd in self.measurements], + time=[ctd.spacetime.time for ctd in self.measurements], max_depth=max_depths, - min_depth=[ctd.min_depth for ctd in ctds], - winch_speed=[WINCH_SPEED for _ in ctds], + min_depth=[ctd.min_depth for ctd in self.measurements], + winch_speed=[WINCH_SPEED for _ in self.measurements], ) # define output file for the simulation - out_file = ctd_particleset.ParticleFile(name=out_path, outputdt=outputdt) + out_file = ctd_particleset.ParticleFile(name=self.out_path, outputdt=OUTPUT_DT) # execute simulation ctd_particleset.execute( diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 85bf02f5e..b017e0f4e 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -1,6 +1,5 @@ from dataclasses import dataclass from datetime import timedelta -from pathlib import Path from typing import ClassVar import numpy as np @@ -109,42 +108,42 @@ def get_datasets_dict(self) -> dict: "o2data": { "dataset_id": "cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m", "variables": ["o2"], - "output_filename": "ctd_bgc_o2.nc", + "output_filename": f"{self.name}_o2.nc", }, "chlorodata": { "dataset_id": "cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m", "variables": ["chl"], - "output_filename": "ctd_bgc_chl.nc", + "output_filename": f"{self.name}_chl.nc", }, "nitratedata": { "dataset_id": "cmems_mod_glo_bgc-nut_anfc_0.25deg_P1D-m", "variables": ["no3"], - "output_filename": "ctd_bgc_no3.nc", + "output_filename": f"{self.name}_no3.nc", }, "phosphatedata": { "dataset_id": "cmems_mod_glo_bgc-nut_anfc_0.25deg_P1D-m", "variables": ["po4"], - "output_filename": "ctd_bgc_po4.nc", + "output_filename": f"{self.name}_po4.nc", }, "phdata": { "dataset_id": "cmems_mod_glo_bgc-car_anfc_0.25deg_P1D-m", "variables": ["ph"], - "output_filename": "ctd_bgc_ph.nc", + "output_filename": f"{self.name}_ph.nc", }, "phytoplanktondata": { "dataset_id": "cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m", "variables": ["phyc"], - "output_filename": "ctd_bgc_phyc.nc", + "output_filename": f"{self.name}_phyc.nc", }, "zooplanktondata": { "dataset_id": "cmems_mod_glo_bgc-plankton_anfc_0.25deg_P1D-m", "variables": ["zooc"], - "output_filename": "ctd_bgc_zooc.nc", + "output_filename": f"{self.name}_zooc.nc", }, "primaryproductiondata": { "dataset_id": "cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m", "variables": ["nppv"], - "output_filename": "ctd_bgc_nppv.nc", + "output_filename": f"{self.name}_nppv.nc", }, } @@ -153,20 +152,17 @@ def get_datasets_dict(self) -> dict: class CTD_BGCInstrument(Instrument): """CTD_BGC instrument class.""" - def __init__( - self, - input_dataset: InputDataset, - ): + def __init__(self, name, expedition, directory): """Initialize CTD_BGCInstrument.""" filenames = { - "o2": input_dataset.data_dir.joinpath("ctd_bgc_o2.nc"), - "chl": input_dataset.data_dir.joinpath("ctd_bgc_chl.nc"), - "no3": input_dataset.data_dir.joinpath("ctd_bgc_no3.nc"), - "po4": input_dataset.data_dir.joinpath("ctd_bgc_po4.nc"), - "ph": input_dataset.data_dir.joinpath("ctd_bgc_ph.nc"), - "phyc": input_dataset.data_dir.joinpath("ctd_bgc_phyc.nc"), - "zooc": input_dataset.data_dir.joinpath("ctd_bgc_zooc.nc"), - "nppv": input_dataset.data_dir.joinpath("ctd_bgc_nppv.nc"), + "o2": directory.joinpath(f"{name}_o2.nc"), + "chl": directory.joinpath(f"{name}_chl.nc"), + "no3": directory.joinpath(f"{name}_no3.nc"), + "po4": directory.joinpath(f"{name}_po4.nc"), + "ph": directory.joinpath(f"{name}_ph.nc"), + "phyc": directory.joinpath(f"{name}_phyc.nc"), + "zooc": directory.joinpath(f"{name}_zooc.nc"), + "nppv": directory.joinpath(f"{name}_nppv.nc"), } variables = { "o2": "o2", @@ -179,21 +175,22 @@ def __init__( "nppv": "nppv", } super().__init__( - input_dataset, + CTD_BGC.name, + expedition, + directory, filenames, variables, add_bathymetry=True, allow_time_extrapolation=True, ) - def simulate( - self, ctd_bgcs: list[CTD_BGC], out_path: str | Path, outputdt: timedelta - ) -> None: + def simulate(self) -> None: """Simulate BGC CTD measurements using Parcels.""" WINCH_SPEED = 1.0 # sink and rise speed in m/s - DT = 10.0 # dt of CTD simulation integrator + DT = 10.0 # dt of CTD_BGC simulation integrator + OUTPUT_DT = timedelta(seconds=10) # output dt for CTD_BGC simulation - if len(ctd_bgcs) == 0: + if len(self.measurements) == 0: print( "No BGC CTDs provided. Parcels currently crashes when providing an empty particle set, so no BGC CTD simulation will be done and no files will be created." ) @@ -209,7 +206,7 @@ def simulate( if not all( [ np.datetime64(ctd_bgc.spacetime.time) >= fieldset_starttime - for ctd_bgc in ctd_bgcs + for ctd_bgc in self.measurements ] ): raise ValueError("BGC CTD deployed before fieldset starts.") @@ -225,7 +222,7 @@ def simulate( time=0, ), ) - for ctd_bgc in ctd_bgcs + for ctd_bgc in self.measurements ] # CTD depth can not be too shallow, because kernel would break. @@ -239,17 +236,19 @@ def simulate( ctd_bgc_particleset = ParticleSet( fieldset=fieldset, pclass=_CTD_BGCParticle, - lon=[ctd_bgc.spacetime.location.lon for ctd_bgc in ctd_bgcs], - lat=[ctd_bgc.spacetime.location.lat for ctd_bgc in ctd_bgcs], - depth=[ctd_bgc.min_depth for ctd_bgc in ctd_bgcs], - time=[ctd_bgc.spacetime.time for ctd_bgc in ctd_bgcs], + lon=[ctd_bgc.spacetime.location.lon for ctd_bgc in self.measurements], + lat=[ctd_bgc.spacetime.location.lat for ctd_bgc in self.measurements], + depth=[ctd_bgc.min_depth for ctd_bgc in self.measurements], + time=[ctd_bgc.spacetime.time for ctd_bgc in self.measurements], max_depth=max_depths, - min_depth=[ctd_bgc.min_depth for ctd_bgc in ctd_bgcs], - winch_speed=[WINCH_SPEED for _ in ctd_bgcs], + min_depth=[ctd_bgc.min_depth for ctd_bgc in self.measurements], + winch_speed=[WINCH_SPEED for _ in self.measurements], ) # define output file for the simulation - out_file = ctd_bgc_particleset.ParticleFile(name=out_path, outputdt=outputdt) + out_file = ctd_bgc_particleset.ParticleFile( + name=self.out_path, outputdt=OUTPUT_DT + ) # execute simulation ctd_bgc_particleset.execute( diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index 186383ce8..72db065ed 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -1,6 +1,5 @@ from dataclasses import dataclass -from datetime import datetime, timedelta -from pathlib import Path +from datetime import timedelta from typing import ClassVar import numpy as np @@ -73,12 +72,12 @@ def get_datasets_dict(self) -> dict: "UVdata": { "dataset_id": "cmems_mod_glo_phy-cur_anfc_0.083deg_PT6H-i", "variables": ["uo", "vo"], - "output_filename": "drifter_uv.nc", + "output_filename": f"{self.name}_uv.nc", }, "Tdata": { "dataset_id": "cmems_mod_glo_phy-thetao_anfc_0.083deg_PT6H-i", "variables": ["thetao"], - "output_filename": "drifter_t.nc", + "output_filename": f"{self.name}_t.nc", }, } @@ -87,34 +86,30 @@ def get_datasets_dict(self) -> dict: class DrifterInstrument(Instrument): """Drifter instrument class.""" - def __init__( - self, - input_dataset: InputDataset, - ): + def __init__(self, name, expedition, directory): """Initialize DrifterInstrument.""" filenames = { - "UV": input_dataset.data_dir.joinpath("drifter_uv.nc"), - "T": input_dataset.data_dir.joinpath("drifter_t.nc"), + "UV": directory.joinpath(f"{name}_uv.nc"), + "T": directory.joinpath(f"{name}_t.nc"), } variables = {"UV": ["uo", "vo"], "T": "thetao"} super().__init__( - input_dataset, + Drifter.name, + expedition, + directory, filenames, variables, add_bathymetry=False, allow_time_extrapolation=False, ) - def simulate( - self, - drifters: list[Drifter], - out_path: str | Path, - outputdt: timedelta, - dt: timedelta, - endtime: datetime | None = None, - ) -> None: + def simulate(self) -> None: """Simulate Drifter measurements.""" - if len(drifters) == 0: + OUTPUT_DT = timedelta(hours=5) + DT = timedelta(minutes=5) + ENDTIME = None + + if len(self.measurements) == 0: print( "No drifters provided. Parcels currently crashes when providing an empty particle set, so no drifter simulation will be done and no files will be created." ) @@ -127,46 +122,49 @@ def simulate( drifter_particleset = ParticleSet( fieldset=fieldset, pclass=_DrifterParticle, - lat=[drifter.spacetime.location.lat for drifter in drifters], - lon=[drifter.spacetime.location.lon for drifter in drifters], - depth=[drifter.depth for drifter in drifters], - time=[drifter.spacetime.time for drifter in drifters], + lat=[drifter.spacetime.location.lat for drifter in self.measurements], + lon=[drifter.spacetime.location.lon for drifter in self.measurements], + depth=[drifter.depth for drifter in self.measurements], + time=[drifter.spacetime.time for drifter in self.measurements], has_lifetime=[ - 1 if drifter.lifetime is not None else 0 for drifter in drifters + 1 if drifter.lifetime is not None else 0 + for drifter in self.measurements ], lifetime=[ 0 if drifter.lifetime is None else drifter.lifetime.total_seconds() - for drifter in drifters + for drifter in self.measurements ], ) # define output file for the simulation out_file = drifter_particleset.ParticleFile( - name=out_path, outputdt=outputdt, chunks=[len(drifter_particleset), 100] + name=self.out_path, + outputdt=OUTPUT_DT, + chunks=[len(drifter_particleset), 100], ) # get earliest between fieldset end time and provide end time fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) - if endtime is None: + if ENDTIME is None: actual_endtime = fieldset_endtime - elif endtime > fieldset_endtime: + elif ENDTIME > fieldset_endtime: print("WARN: Requested end time later than fieldset end time.") actual_endtime = fieldset_endtime else: - actual_endtime = np.timedelta64(endtime) + actual_endtime = np.timedelta64(ENDTIME) # execute simulation drifter_particleset.execute( [AdvectionRK4, _sample_temperature, _check_lifetime], endtime=actual_endtime, - dt=dt, + dt=DT, output_file=out_file, verbose_progress=True, ) # if there are more particles left than the number of drifters with an indefinite endtime, warn the user if len(drifter_particleset.particledata) > len( - [d for d in drifters if d.lifetime is None] + [d for d in self.measurements if d.lifetime is None] ): print( "WARN: Some drifters had a life time beyond the end time of the fieldset or the requested end time." diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index 371b4485a..9c0fc4011 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -1,5 +1,4 @@ from dataclasses import dataclass -from pathlib import Path from typing import ClassVar import numpy as np @@ -7,7 +6,6 @@ from parcels import ParticleSet, ScipyParticle, Variable from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType -from virtualship.models.spacetime import Spacetime from virtualship.utils import register_input_dataset, register_instrument @@ -80,32 +78,28 @@ def get_datasets_dict(self) -> dict: class Underwater_STInstrument(Instrument): """Underwater_ST instrument class.""" - def __init__( - self, - input_dataset: InputDataset, - ): + def __init__(self, name, expedition, directory): """Initialize Underwater_STInstrument.""" filenames = { - "S": input_dataset.data_dir.joinpath(f"{input_dataset.name}_s.nc"), - "T": input_dataset.data_dir.joinpath(f"{input_dataset.name}_t.nc"), + "S": directory.joinpath(f"{name}_s.nc"), + "T": directory.joinpath(f"{name}_t.nc"), } variables = {"S": "so", "T": "thetao"} super().__init__( - input_dataset, + Underwater_ST.name, + expedition, + directory, filenames, variables, add_bathymetry=False, allow_time_extrapolation=True, ) - def simulate( - self, - out_path: str | Path, - depth: float, - sample_points: list[Spacetime], - ) -> None: + def simulate(self) -> None: """Simulate underway salinity and temperature measurements.""" - sample_points.sort(key=lambda p: p.time) + DEPTH = -2.0 + + self.measurements.sort(key=lambda p: p.time) fieldset = self.load_input_data() @@ -114,13 +108,13 @@ def simulate( pclass=_ShipSTParticle, lon=0.0, lat=0.0, - depth=depth, + depth=DEPTH, time=0, ) - out_file = particleset.ParticleFile(name=out_path, outputdt=np.inf) + out_file = particleset.ParticleFile(name=self.out_path, outputdt=np.inf) - for point in sample_points: + for point in self.measurements: particleset.lon_nextloop[:] = point.location.lon particleset.lat_nextloop[:] = point.location.lat particleset.time_nextloop[:] = fieldset.time_origin.reltime( diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index c2fec98d3..7c916d98e 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -1,6 +1,5 @@ from dataclasses import dataclass from datetime import timedelta -from pathlib import Path from typing import ClassVar import numpy as np @@ -87,17 +86,17 @@ def get_datasets_dict(self) -> dict: "UVdata": { "dataset_id": "cmems_mod_glo_phy-cur_anfc_0.083deg_PT6H-i", "variables": ["uo", "vo"], - "output_filename": "ship_uv.nc", + "output_filename": f"{self.name}_uv.nc", }, "Sdata": { "dataset_id": "cmems_mod_glo_phy-so_anfc_0.083deg_PT6H-i", "variables": ["so"], - "output_filename": "ship_s.nc", + "output_filename": f"{self.name}_s.nc", }, "Tdata": { "dataset_id": "cmems_mod_glo_phy-thetao_anfc_0.083deg_PT6H-i", "variables": ["thetao"], - "output_filename": "ship_t.nc", + "output_filename": f"{self.name}_t.nc", }, } @@ -106,35 +105,30 @@ def get_datasets_dict(self) -> dict: class XBTInstrument(Instrument): """XBT instrument class.""" - def __init__( - self, - input_dataset: InputDataset, - ): + def __init__(self, name, expedition, directory): """Initialize XBTInstrument.""" filenames = { - "UV": input_dataset.data_dir.joinpath("ship_uv.nc"), - "S": input_dataset.data_dir.joinpath("ship_s.nc"), - "T": input_dataset.data_dir.joinpath("ship_t.nc"), + "UV": directory.joinpath(f"{name}_uv.nc"), + "S": directory.joinpath(f"{name}_s.nc"), + "T": directory.joinpath(f"{name}_t.nc"), } variables = {"UV": ["uo", "vo"], "S": "so", "T": "thetao"} super().__init__( - input_dataset, + XBT.name, + expedition, + directory, filenames, variables, add_bathymetry=True, allow_time_extrapolation=True, ) - def simulate( - self, - xbts: list[XBT], - out_path: str | Path, - outputdt: timedelta, - ) -> None: + def simulate(self) -> None: """Simulate XBT measurements.""" DT = 10.0 # dt of XBT simulation integrator + OUTPUT_DT = timedelta(seconds=1) - if len(xbts) == 0: + if len(self.measurements) == 0: print( "No XBTs provided. Parcels currently crashes when providing an empty particle set, so no XBT simulation will be done and no files will be created." ) @@ -148,7 +142,10 @@ def simulate( # deploy time for all xbts should be later than fieldset start time if not all( - [np.datetime64(xbt.spacetime.time) >= fieldset_starttime for xbt in xbts] + [ + np.datetime64(xbt.spacetime.time) >= fieldset_starttime + for xbt in self.measurements + ] ): raise ValueError("XBT deployed before fieldset starts.") @@ -163,11 +160,11 @@ def simulate( time=0, ), ) - for xbt in xbts + for xbt in self.measurements ] # initial fall speeds - initial_fall_speeds = [xbt.fall_speed for xbt in xbts] + initial_fall_speeds = [xbt.fall_speed for xbt in self.measurements] # XBT depth can not be too shallow, because kernel would break. for max_depth, fall_speed in zip(max_depths, initial_fall_speeds, strict=False): @@ -180,16 +177,16 @@ def simulate( xbt_particleset = ParticleSet( fieldset=fieldset, pclass=_XBTParticle, - lon=[xbt.spacetime.location.lon for xbt in xbts], - lat=[xbt.spacetime.location.lat for xbt in xbts], - depth=[xbt.min_depth for xbt in xbts], - time=[xbt.spacetime.time for xbt in xbts], + lon=[xbt.spacetime.location.lon for xbt in self.measurements], + lat=[xbt.spacetime.location.lat for xbt in self.measurements], + depth=[xbt.min_depth for xbt in self.measurements], + time=[xbt.spacetime.time for xbt in self.measurements], max_depth=max_depths, - min_depth=[xbt.min_depth for xbt in xbts], - fall_speed=[xbt.fall_speed for xbt in xbts], + min_depth=[xbt.min_depth for xbt in self.measurements], + fall_speed=[xbt.fall_speed for xbt in self.measurements], ) - out_file = xbt_particleset.ParticleFile(name=out_path, outputdt=outputdt) + out_file = xbt_particleset.ParticleFile(name=self.out_path, outputdt=OUTPUT_DT) xbt_particleset.execute( [_sample_temperature, _xbt_cast], From 3efa29bda70de2e3a22a287321142033c2fb5a7f Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 28 Oct 2025 14:24:53 +0100 Subject: [PATCH 30/97] draft up check land using bathymetry --- src/virtualship/expedition/do_expedition.py | 4 +- src/virtualship/instruments/base.py | 6 -- src/virtualship/models/expedition.py | 70 +++++++++------------ 3 files changed, 31 insertions(+), 49 deletions(-) diff --git a/src/virtualship/expedition/do_expedition.py b/src/virtualship/expedition/do_expedition.py index c4bc67837..7be947b28 100644 --- a/src/virtualship/expedition/do_expedition.py +++ b/src/virtualship/expedition/do_expedition.py @@ -48,10 +48,8 @@ def do_expedition(expedition_dir: str | Path, input_data: Path | None = None) -> # verify schedule is valid # TODO: needs updating when .verify() updated to not need input_data - - loaded_input_data = [] # TODO: TEMPORARY! expedition.schedule.verify( - expedition.ship_config.ship_speed_knots, loaded_input_data + expedition.ship_config.ship_speed_knots, input_dir=input_data ) # simulate the schedule diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 13a2fb055..001dc8c5d 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -99,14 +99,8 @@ def __init__( def load_input_data(self) -> FieldSet: """Load and return the input data as a FieldSet for the instrument.""" - # TODO: this should mean can delete input_data.py! - - # TODO: hopefully simulate_measurements can also be removed! And maybe the list of e.g. ctds ('measurements') to run can be added to higher level like do_expedition.py...? I think as they already do... - # TODO: can simulate_schedule.py be refactored to be contained in base.py and repsective instrument files too...? - # TODO: what do I need to do about automatic registration of Instrument classes...? - # TODO: tests will need updating...! # TODO: think about combining InputDataset and Instrument classes together! Or maybe not if they are better kept separate... diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index 9add2c1bd..8888f6572 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -2,12 +2,15 @@ import itertools from datetime import datetime, timedelta +from pathlib import Path from typing import TYPE_CHECKING +import numpy as np import pydantic import pyproj import yaml +from parcels import Field from virtualship.errors import InstrumentsConfigError, ScheduleError from virtualship.instruments.types import InstrumentType from virtualship.utils import _validate_numeric_mins_to_timedelta @@ -87,9 +90,9 @@ class Schedule(pydantic.BaseModel): def verify( self, ship_speed: float, + input_dir: str | Path | None, *, check_space_time_region: bool = False, - ignore_missing_fieldsets: bool = False, ) -> None: """ Verify the feasibility and correctness of the schedule's waypoints. @@ -102,9 +105,9 @@ def verify( 5. The ship can arrive on time at each waypoint given its speed. :param ship_speed: The ship's speed in knots. - :param input_data: An InputData object containing fieldsets used to check if waypoints are on water. + :param input_dir: The input directory containing necessary files. :param check_space_time_region: whether to check for missing space_time_region. - :param ignore_missing_fieldsets: whether to ignore warning for missing field sets. + # :param ignore_missing_fieldsets: whether to ignore warning for missing field sets. :raises PlanningError: If any of the verification checks fail, indicating infeasible or incorrect waypoints. :raises NotImplementedError: If an instrument in the schedule is not implemented. :return: None. The method doesn't return a value but raises exceptions if verification fails. @@ -134,46 +137,33 @@ def verify( f"Waypoint(s) {', '.join(f'#{i + 1}' for i in invalid_i)}: each waypoint should be timed after all previous waypoints", ) - # check if all waypoints are in water - # this is done by picking an arbitrary provided fieldset and checking if UV is not zero - - # TODO: this may need to be done with generic bathymetry data, now that removed InputData! - # get all available fieldsets - available_fieldsets = [] - # if input_data is not None: - # fieldsets = [ - # input_data.adcp_fieldset, - # input_data.argo_float_fieldset, - # input_data.ctd_fieldset, - # input_data.drifter_fieldset, - # input_data.ship_underwater_st_fieldset, - # ] - # for fs in fieldsets: - # if fs is not None: - # available_fieldsets.append(fs) - - # check if there are any fieldsets, else it's an error - if len(available_fieldsets) == 0: - if not ignore_missing_fieldsets: - print( - "Cannot verify because no fieldsets have been loaded. This is probably " - "because you are not using any instruments in your schedule. This is not a problem, " - "but carefully check your waypoint locations manually." + # check if all waypoints are in water using bathymetry data + # TODO: tests should be updated to check this! + land_waypoints = [] + if input_dir is not None: + bathymetry_path = input_dir.joinpath("bathymetry.nc") + try: + bathymetry_field = Field.from_netcdf( + bathymetry_path, + variables=("bathymetry", "deptho"), + dimensions={"lon": "longitude", "lat": "latitude"}, ) - - else: - # pick any - fieldset = available_fieldsets[0] - # get waypoints with 0 UV - land_waypoints = [ - (wp_i, wp) - for wp_i, wp in enumerate(self.waypoints) - if _is_on_land_zero_uv(fieldset, wp) - ] - # raise an error if there are any + except Exception as e: + raise FileNotFoundError( + "Bathymetry file not found in input data. Cannot verify waypoints are in water." + ) from e + for wp_i, wp in enumerate(self.waypoints): + bathy = bathymetry_field.eval( + 0, # time + 0, # depth (surface) + wp.location.lat, + wp.location.lon, + ) + if np.isnan(bathy) or bathy >= 0: + land_waypoints.append((wp_i, wp)) if len(land_waypoints) > 0: raise ScheduleError( - f"The following waypoints are on land: {['#' + str(wp_i) + ' ' + str(wp) for (wp_i, wp) in land_waypoints]}" + f"The following waypoints are on land: {['#' + str(wp_i + 1) + ' ' + str(wp) for (wp_i, wp) in land_waypoints]}" ) # check that ship will arrive on time at each waypoint (in case no unexpected event happen) From aa2d3095be32b63695e146e7dedfcdb2c3985eef Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 28 Oct 2025 14:55:44 +0100 Subject: [PATCH 31/97] small bug fixes --- src/virtualship/cli/_fetch.py | 3 +-- src/virtualship/cli/_plan.py | 7 ++++--- src/virtualship/expedition/__init__.py | 2 -- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/virtualship/cli/_fetch.py b/src/virtualship/cli/_fetch.py index 67695695e..e472eb9ad 100644 --- a/src/virtualship/cli/_fetch.py +++ b/src/virtualship/cli/_fetch.py @@ -52,9 +52,8 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None expedition.schedule.verify( expedition.ship_config.ship_speed_knots, - input_data=None, + input_dir=None, check_space_time_region=True, - ignore_missing_fieldsets=True, ) space_time_region_hash = get_space_time_region_hash( diff --git a/src/virtualship/cli/_plan.py b/src/virtualship/cli/_plan.py index a071c38e7..41709a5c5 100644 --- a/src/virtualship/cli/_plan.py +++ b/src/virtualship/cli/_plan.py @@ -951,7 +951,9 @@ def copy_from_previous(self) -> None: if prev and curr: curr.value = prev.value - for instrument in InstrumentType: + for instrument in [ + inst for inst in InstrumentType if not inst.is_underway + ]: prev_switch = schedule_editor.query_one( f"#wp{self.index - 1}_{instrument.value}" ) @@ -1044,9 +1046,8 @@ def save_pressed(self) -> None: # verify schedule expedition_editor.expedition.schedule.verify( ship_speed_value, - input_data=None, + input_dir=None, check_space_time_region=True, - ignore_missing_fieldsets=True, ) expedition_saved = expedition_editor.save_changes() diff --git a/src/virtualship/expedition/__init__.py b/src/virtualship/expedition/__init__.py index 43d248448..dfa610283 100644 --- a/src/virtualship/expedition/__init__.py +++ b/src/virtualship/expedition/__init__.py @@ -1,9 +1,7 @@ """Everything for simulating an expedition.""" from .do_expedition import do_expedition -from .input_data import InputData __all__ = [ - "InputData", "do_expedition", ] From 588cab4fc6fde2eee6c3e73c80b53f778786b54e Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 29 Oct 2025 11:23:28 +0100 Subject: [PATCH 32/97] patch copernicus product id search logic to new instrument classes, plus more debugging; verbose INFO is outstanding --- src/virtualship/cli/_fetch.py | 141 +------------ src/virtualship/cli/_plan.py | 3 +- src/virtualship/expedition/do_expedition.py | 29 ++- .../expedition/simulate_schedule.py | 22 ++- src/virtualship/instruments/adcp.py | 23 ++- src/virtualship/instruments/argo_float.py | 43 ++-- src/virtualship/instruments/base.py | 185 ++++++++++++++++-- src/virtualship/instruments/ctd.py | 23 +-- src/virtualship/instruments/ctd_bgc.py | 47 ++--- src/virtualship/instruments/drifter.py | 32 +-- .../instruments/ship_underwater_st.py | 19 +- src/virtualship/instruments/xbt.py | 27 +-- src/virtualship/models/expedition.py | 32 +-- 13 files changed, 339 insertions(+), 287 deletions(-) diff --git a/src/virtualship/cli/_fetch.py b/src/virtualship/cli/_fetch.py index e472eb9ad..a41687d00 100644 --- a/src/virtualship/cli/_fetch.py +++ b/src/virtualship/cli/_fetch.py @@ -7,11 +7,10 @@ from typing import TYPE_CHECKING import copernicusmarine -import numpy as np from copernicusmarine.core_functions.credentials_utils import InvalidUsernameOrPassword from pydantic import BaseModel -from virtualship.errors import CopernicusCatalogueError, IncompleteDownloadError +from virtualship.errors import IncompleteDownloadError from virtualship.utils import ( _dump_yaml, _generic_load_yaml, @@ -52,8 +51,9 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None expedition.schedule.verify( expedition.ship_config.ship_speed_knots, - input_dir=None, + data_dir=None, check_space_time_region=True, + ignore_missing_bathymetry=True, ) space_time_region_hash = get_space_time_region_hash( @@ -107,7 +107,7 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None coordinates_selection_method="outside", ) - # Only keep instruments present in the expedition + # download, only instruments present in the expedition for itype in instruments_in_expedition: input_dataset_class = get_input_dataset_class(itype) if input_dataset_class is None: @@ -225,136 +225,3 @@ def complete_download(download_path: Path) -> None: metadata = DownloadMetadata(download_complete=True, download_date=datetime.now()) metadata.to_yaml(download_metadata) return - - -def select_product_id( - physical: bool, - schedule_start: datetime, - schedule_end: datetime, - username: str, - password: str, - variable: str | None = None, # only needed for BGC datasets -) -> str: - """ - Determine which copernicus product id should be selected (reanalysis, reanalysis-interim, analysis & forecast), for prescribed schedule and physical vs. BGC. - - BGC is more complicated than physical products. Often (re)analysis period and variable dependent, hence more custom logic here. - """ - product_ids = { - "phys": { - "reanalysis": "cmems_mod_glo_phy_my_0.083deg_P1D-m", - "reanalysis_interim": "cmems_mod_glo_phy_myint_0.083deg_P1D-m", - "analysis": "cmems_mod_glo_phy_anfc_0.083deg_P1D-m", - }, - "bgc": { - "reanalysis": "cmems_mod_glo_bgc_my_0.25deg_P1D-m", - "reanalysis_interim": "cmems_mod_glo_bgc_myint_0.25deg_P1D-m", - "analysis": None, # will be set per variable - }, - } - - bgc_analysis_ids = { - "o2": "cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m", - "chl": "cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m", - "no3": "cmems_mod_glo_bgc-nut_anfc_0.25deg_P1D-m", - "po4": "cmems_mod_glo_bgc-nut_anfc_0.25deg_P1D-m", - "ph": "cmems_mod_glo_bgc-car_anfc_0.25deg_P1D-m", - "phyc": "cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m", - "nppv": "cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m", - } - - # pH and phytoplankton variables are available as *monthly* products only in renalysis(_interim) period - monthly_bgc_reanalysis_ids = { - "ph": "cmems_mod_glo_bgc_my_0.25deg_P1M-m", - "phyc": "cmems_mod_glo_bgc_my_0.25deg_P1M-m", - } - monthly_bgc_reanalysis_interim_ids = { - "ph": "cmems_mod_glo_bgc_myint_0.25deg_P1M-m", - "phyc": "cmems_mod_glo_bgc_myint_0.25deg_P1M-m", - } - - key = "phys" if physical else "bgc" - selected_id = None - - for period, pid in product_ids[key].items(): - # for BGC analysis, set pid per variable - if key == "bgc" and period == "analysis": - if variable is None or variable not in bgc_analysis_ids: - continue - pid = bgc_analysis_ids[variable] - # for BGC reanalysis, check if requires monthly product - if ( - key == "bgc" - and period == "reanalysis" - and variable in monthly_bgc_reanalysis_ids - ): - monthly_pid = monthly_bgc_reanalysis_ids[variable] - ds_monthly = copernicusmarine.open_dataset( - monthly_pid, - username=username, - password=password, - ) - time_end_monthly = ds_monthly["time"][-1].values - if np.datetime64(schedule_end) <= time_end_monthly: - pid = monthly_pid - # for BGC reanalysis_interim, check if requires monthly product - if ( - key == "bgc" - and period == "reanalysis_interim" - and variable in monthly_bgc_reanalysis_interim_ids - ): - monthly_pid = monthly_bgc_reanalysis_interim_ids[variable] - ds_monthly = copernicusmarine.open_dataset( - monthly_pid, username=username, password=password - ) - time_end_monthly = ds_monthly["time"][-1].values - if np.datetime64(schedule_end) <= time_end_monthly: - pid = monthly_pid - if pid is None: - continue - ds = copernicusmarine.open_dataset(pid, username=username, password=password) - time_end = ds["time"][-1].values - if np.datetime64(schedule_end) <= time_end: - selected_id = pid - break - - if selected_id is None: - raise CopernicusCatalogueError( - "No suitable product found in the Copernicus Marine Catalogue for the scheduled time and variable." - ) - - # handle the rare situation where start time and end time span different products, which is possible for reanalysis and reanalysis_interim - # in this case, return the analysis product which spans far back enough - if start_end_in_product_timerange( - selected_id, schedule_start, schedule_end, username, password - ): - return selected_id - - else: - return ( - product_ids["phys"]["analysis"] if physical else bgc_analysis_ids[variable] - ) - - -def start_end_in_product_timerange( - selected_id: str, - schedule_start: datetime, - schedule_end: datetime, - username: str, - password: str, -) -> bool: - """Check schedule_start and schedule_end are both within a selected Copernicus product's time range.""" - ds_selected = copernicusmarine.open_dataset( - selected_id, username=username, password=password - ) - time_values = ds_selected["time"].values - time_min, time_max = np.min(time_values), np.max(time_values) - - if ( - np.datetime64(schedule_start) >= time_min - and np.datetime64(schedule_end) <= time_max - ): - return True - - else: - return False diff --git a/src/virtualship/cli/_plan.py b/src/virtualship/cli/_plan.py index 41709a5c5..c27cb7412 100644 --- a/src/virtualship/cli/_plan.py +++ b/src/virtualship/cli/_plan.py @@ -1046,8 +1046,9 @@ def save_pressed(self) -> None: # verify schedule expedition_editor.expedition.schedule.verify( ship_speed_value, - input_dir=None, + data_dir=None, check_space_time_region=True, + ignore_missing_bathymetry=True, ) expedition_saved = expedition_editor.save_changes() diff --git a/src/virtualship/expedition/do_expedition.py b/src/virtualship/expedition/do_expedition.py index 7be947b28..ec61b1b08 100644 --- a/src/virtualship/expedition/do_expedition.py +++ b/src/virtualship/expedition/do_expedition.py @@ -6,23 +6,31 @@ import pyproj +from virtualship.cli._fetch import get_existing_download, get_space_time_region_hash from virtualship.models import Schedule -from virtualship.utils import CHECKPOINT, _get_expedition, get_instrument_class +from virtualship.utils import ( + CHECKPOINT, + _get_expedition, + get_instrument_class, +) from .checkpoint import Checkpoint from .expedition_cost import expedition_cost -from .simulate_schedule import ScheduleProblem, simulate_schedule +from .simulate_schedule import ( + MeasurementsToSimulate, + ScheduleProblem, + simulate_schedule, +) # projection used to sail between waypoints projection = pyproj.Geod(ellps="WGS84") -def do_expedition(expedition_dir: str | Path, input_data: Path | None = None) -> None: +def do_expedition(expedition_dir: str | Path) -> None: """ Perform an expedition, providing terminal feedback and file output. :param expedition_dir: The base directory for the expedition. - :param input_data: Input data folder (override used for testing). """ print("\n╔═════════════════════════════════════════════════╗") print("║ VIRTUALSHIP EXPEDITION STATUS ║") @@ -47,11 +55,13 @@ def do_expedition(expedition_dir: str | Path, input_data: Path | None = None) -> print("\n---- WAYPOINT VERIFICATION ----") # verify schedule is valid - # TODO: needs updating when .verify() updated to not need input_data - expedition.schedule.verify( - expedition.ship_config.ship_speed_knots, input_dir=input_data + data_dir = get_existing_download( + expedition_dir, + get_space_time_region_hash(expedition.schedule.space_time_region), ) + expedition.schedule.verify(expedition.ship_config.ship_speed_knots, data_dir) + # simulate the schedule schedule_results = simulate_schedule( projection=projection, @@ -96,8 +106,9 @@ def do_expedition(expedition_dir: str | Path, input_data: Path | None = None) -> if instrument_class is None: raise RuntimeError(f"No instrument class found for type {itype}.") - # get measurements to simulate for this instrument - measurements = schedule_results.measurements_to_simulate.get(itype.name.lower()) + # get measurements to simulate + attr = MeasurementsToSimulate.get_attr_for_instrumenttype(itype) + measurements = getattr(schedule_results.measurements_to_simulate, attr) # initialise instrument instrument = instrument_class(expedition=expedition, directory=expedition_dir) diff --git a/src/virtualship/expedition/simulate_schedule.py b/src/virtualship/expedition/simulate_schedule.py index f8d142eac..3b8fa78a1 100644 --- a/src/virtualship/expedition/simulate_schedule.py +++ b/src/virtualship/expedition/simulate_schedule.py @@ -4,6 +4,7 @@ from dataclasses import dataclass, field from datetime import datetime, timedelta +from typing import ClassVar import pyproj @@ -39,7 +40,26 @@ class ScheduleProblem: @dataclass class MeasurementsToSimulate: - """The measurements to simulate, as concluded from schedule simulation.""" + """ + The measurements to simulate, as concluded from schedule simulation. + + Provides a mapping from InstrumentType to the correct attribute name for robust access. + """ + + _instrumenttype_to_attr: ClassVar[dict] = { + InstrumentType.ADCP: "adcps", + InstrumentType.UNDERWATER_ST: "ship_underwater_sts", + InstrumentType.ARGO_FLOAT: "argo_floats", + InstrumentType.DRIFTER: "drifters", + InstrumentType.CTD: "ctds", + InstrumentType.CTD_BGC: "ctd_bgcs", + InstrumentType.XBT: "xbts", + } + + @classmethod + def get_attr_for_instrumenttype(cls, instrument_type): + """Return the attribute name for a given InstrumentType.""" + return cls._instrumenttype_to_attr[instrument_type] adcps: list[Spacetime] = field(default_factory=list, init=False) ship_underwater_sts: list[Spacetime] = field(default_factory=list, init=False) diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index 917da1548..db5be8eff 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -63,7 +63,7 @@ def get_datasets_dict(self) -> dict: """Get variable specific args for instrument.""" return { "UVdata": { - "dataset_id": "cmems_mod_glo_phy-cur_anfc_0.083deg_PT6H-i", + "physical": True, "variables": ["uo", "vo"], "output_filename": f"{self.name}_uv.nc", }, @@ -74,12 +74,13 @@ def get_datasets_dict(self) -> dict: class ADCPInstrument(Instrument): """ADCP instrument class.""" - def __init__(self, name, expedition, directory): + def __init__(self, expedition, directory): """Initialize ADCPInstrument.""" filenames = { - "UV": directory.joinpath(f"{name}_uv.nc"), + "U": f"{ADCP.name}_uv.nc", + "V": f"{ADCP.name}_uv.nc", } - variables = {"UV": ["uo", "vo"]} + variables = {"U": "uo", "V": "vo"} super().__init__( ADCP.name, expedition, @@ -90,13 +91,13 @@ def __init__(self, name, expedition, directory): allow_time_extrapolation=True, ) - def simulate(self) -> None: + def simulate(self, measurements, out_path) -> None: """Simulate ADCP measurements.""" MAX_DEPTH = self.expedition.instruments_config.adcp_config.max_depth_meter MIN_DEPTH = -5.0 - NUM_BINS = self.instruments_config.adcp_config.num_bins + NUM_BINS = self.expedition.instruments_config.adcp_config.num_bins - self.measurements.sort(key=lambda p: p.time) + measurements.sort(key=lambda p: p.time) fieldset = self.load_input_data() @@ -105,15 +106,17 @@ def simulate(self) -> None: particleset = ParticleSet.from_list( fieldset=fieldset, pclass=_ADCPParticle, - lon=np.full(num_particles, 0.0), + lon=np.full( + num_particles, 0.0 + ), # initial lat/lon are irrelevant and will be overruled later.s lat=np.full(num_particles, 0.0), depth=bins, time=0, ) - out_file = particleset.ParticleFile(name=self.out_path, outputdt=np.inf) + out_file = particleset.ParticleFile(name=out_path, outputdt=np.inf) - for point in self.measurements: + for point in measurements: particleset.lon_nextloop[:] = point.location.lon particleset.lat_nextloop[:] = point.location.lat particleset.time_nextloop[:] = fieldset.time_origin.reltime( diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index 239b47de6..93aa90936 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -144,17 +144,17 @@ def get_datasets_dict(self) -> dict: """Get variable specific args for instrument.""" return { "UVdata": { - "dataset_id": "cmems_mod_glo_phy-cur_anfc_0.083deg_PT6H-i", + "physical": True, "variables": ["uo", "vo"], "output_filename": f"{self.name}_uv.nc", }, "Sdata": { - "dataset_id": "cmems_mod_glo_phy-so_anfc_0.083deg_PT6H-i", + "physical": True, "variables": ["so"], "output_filename": f"{self.name}_s.nc", }, "Tdata": { - "dataset_id": "cmems_mod_glo_phy-thetao_anfc_0.083deg_PT6H-i", + "physical": True, "variables": ["thetao"], "output_filename": f"{self.name}_t.nc", }, @@ -165,14 +165,15 @@ def get_datasets_dict(self) -> dict: class ArgoFloatInstrument(Instrument): """ArgoFloat instrument class.""" - def __init__(self, name, expedition, directory): + def __init__(self, expedition, directory): """Initialize ArgoFloatInstrument.""" filenames = { - "UV": directory.joinpath(f"{name}_uv.nc"), - "S": directory.joinpath(f"{name}_s.nc"), - "T": directory.joinpath(f"{name}_t.nc"), + "U": f"{ArgoFloat.name}_uv.nc", + "V": f"{ArgoFloat.name}_uv.nc", + "S": f"{ArgoFloat.name}_s.nc", + "T": f"{ArgoFloat.name}_t.nc", } - variables = {"UV": ["uo", "vo"], "S": "so", "T": "thetao"} + variables = {"U": "uo", "V": "vo", "S": "so", "T": "thetao"} super().__init__( ArgoFloat.name, expedition, @@ -183,13 +184,13 @@ def __init__(self, name, expedition, directory): allow_time_extrapolation=False, ) - def simulate(self) -> None: + def simulate(self, measurements, out_path) -> None: """Simulate Argo float measurements.""" DT = 10.0 # dt of Argo float simulation integrator OUTPUT_DT = timedelta(minutes=5) ENDTIME = None - if len(self.measurements) == 0: + if len(measurements) == 0: print( "No Argo floats provided. Parcels currently crashes when providing an empty particle set, so no argo floats simulation will be done and no files will be created." ) @@ -202,21 +203,21 @@ def simulate(self) -> None: argo_float_particleset = ParticleSet( fieldset=fieldset, pclass=_ArgoParticle, - lat=[argo.spacetime.location.lat for argo in self.measurements], - lon=[argo.spacetime.location.lon for argo in self.measurements], - depth=[argo.min_depth for argo in self.measurements], - time=[argo.spacetime.time for argo in self.measurements], - min_depth=[argo.min_depth for argo in self.measurements], - max_depth=[argo.max_depth for argo in self.measurements], - drift_depth=[argo.drift_depth for argo in self.measurements], - vertical_speed=[argo.vertical_speed for argo in self.measurements], - cycle_days=[argo.cycle_days for argo in self.measurements], - drift_days=[argo.drift_days for argo in self.measurements], + lat=[argo.spacetime.location.lat for argo in measurements], + lon=[argo.spacetime.location.lon for argo in measurements], + depth=[argo.min_depth for argo in measurements], + time=[argo.spacetime.time for argo in measurements], + min_depth=[argo.min_depth for argo in measurements], + max_depth=[argo.max_depth for argo in measurements], + drift_depth=[argo.drift_depth for argo in measurements], + vertical_speed=[argo.vertical_speed for argo in measurements], + cycle_days=[argo.cycle_days for argo in measurements], + drift_days=[argo.drift_days for argo in measurements], ) # define output file for the simulation out_file = argo_float_particleset.ParticleFile( - name=self.out_path, + name=out_path, outputdt=OUTPUT_DT, chunks=[len(argo_float_particleset), 100], ) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 001dc8c5d..6ae38cc0f 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -3,12 +3,47 @@ from pathlib import Path import copernicusmarine -import yaspin +import numpy as np +from yaspin import yaspin from parcels import Field, FieldSet +from virtualship.cli._fetch import get_existing_download, get_space_time_region_hash +from virtualship.errors import CopernicusCatalogueError from virtualship.models import Expedition, SpaceTimeRegion from virtualship.utils import ship_spinner +PRODUCT_IDS = { + "phys": { + "reanalysis": "cmems_mod_glo_phy_my_0.083deg_P1D-m", + "reanalysis_interim": "cmems_mod_glo_phy_myint_0.083deg_P1D-m", + "analysis": "cmems_mod_glo_phy_anfc_0.083deg_P1D-m", + }, + "bgc": { + "reanalysis": "cmems_mod_glo_bgc_my_0.25deg_P1D-m", + "reanalysis_interim": "cmems_mod_glo_bgc_myint_0.25deg_P1D-m", + "analysis": None, # will be set per variable + }, +} + +BGC_ANALYSIS_IDS = { + "o2": "cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m", + "chl": "cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m", + "no3": "cmems_mod_glo_bgc-nut_anfc_0.25deg_P1D-m", + "po4": "cmems_mod_glo_bgc-nut_anfc_0.25deg_P1D-m", + "ph": "cmems_mod_glo_bgc-car_anfc_0.25deg_P1D-m", + "phyc": "cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m", + "nppv": "cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m", +} + +MONTHLY_BGC_REANALYSIS_IDS = { + "ph": "cmems_mod_glo_bgc_my_0.25deg_P1M-m", + "phyc": "cmems_mod_glo_bgc_my_0.25deg_P1M-m", +} +MONTHLY_BGC_REANALYSIS_INTERIM_IDS = { + "ph": "cmems_mod_glo_bgc_myint_0.25deg_P1M-m", + "phyc": "cmems_mod_glo_bgc_myint_0.25deg_P1M-m", +} + class InputDataset(abc.ABC): """Base class for instrument input datasets.""" @@ -39,7 +74,7 @@ def get_datasets_dict(self) -> dict: """Get parameters for instrument's variable(s) specific data download.""" def download_data(self) -> None: - """Download data for the instrument using copernicusmarine.""" + """Download data for the instrument using copernicusmarine, with correct product ID selection.""" parameter_args = dict( minimum_longitude=self.space_time_region.spatial_range.minimum_longitude - self.latlon_buffer, @@ -62,10 +97,117 @@ def download_data(self) -> None: ) datasets_args = self.get_datasets_dict() + for dataset in datasets_args.values(): - download_args = {**parameter_args, **dataset} + physical = dataset.get("physical") + if physical: + variable = None + else: + variable = dataset.get("variables")[0] # BGC variables, special case + + dataset_id = self._select_product_id( + physical=physical, + schedule_start=self.space_time_region.time_range.start_time, + schedule_end=self.space_time_region.time_range.end_time, + username=self.credentials["username"], + password=self.credentials["password"], + variable=variable, + ) + download_args = { + **parameter_args, + **{k: v for k, v in dataset.items() if k != "physical"}, + "dataset_id": dataset_id, + } copernicusmarine.subset(**download_args) + def _select_product_id( + self, + physical: bool, + schedule_start, + schedule_end, + username: str, + password: str, + variable: str | None = None, + ) -> str: + """Determine which copernicus product id should be selected (reanalysis, reanalysis-interim, analysis & forecast), for prescribed schedule and physical vs. BGC.""" + key = "phys" if physical else "bgc" + selected_id = None + + for period, pid in PRODUCT_IDS[key].items(): + # for BGC analysis, set pid per variable + if key == "bgc" and period == "analysis": + if variable is None or variable not in BGC_ANALYSIS_IDS: + continue + pid = BGC_ANALYSIS_IDS[variable] + # for BGC reanalysis, check if requires monthly product + if ( + key == "bgc" + and period == "reanalysis" + and variable in MONTHLY_BGC_REANALYSIS_IDS + ): + monthly_pid = MONTHLY_BGC_REANALYSIS_IDS[variable] + ds_monthly = copernicusmarine.open_dataset( + monthly_pid, + username=username, + password=password, + ) + time_end_monthly = ds_monthly["time"][-1].values + if np.datetime64(schedule_end) <= time_end_monthly: + pid = monthly_pid + # for BGC reanalysis_interim, check if requires monthly product + if ( + key == "bgc" + and period == "reanalysis_interim" + and variable in MONTHLY_BGC_REANALYSIS_INTERIM_IDS + ): + monthly_pid = MONTHLY_BGC_REANALYSIS_INTERIM_IDS[variable] + ds_monthly = copernicusmarine.open_dataset( + monthly_pid, username=username, password=password + ) + time_end_monthly = ds_monthly["time"][-1].values + if np.datetime64(schedule_end) <= time_end_monthly: + pid = monthly_pid + if pid is None: + continue + ds = copernicusmarine.open_dataset( + pid, username=username, password=password + ) + time_end = ds["time"][-1].values + if np.datetime64(schedule_end) <= time_end: + selected_id = pid + break + + if selected_id is None: + raise CopernicusCatalogueError( + "No suitable product found in the Copernicus Marine Catalogue for the scheduled time and variable." + ) + + def start_end_in_product_timerange( + selected_id, schedule_start, schedule_end, username, password + ): + ds_selected = copernicusmarine.open_dataset( + selected_id, username=username, password=password + ) + time_values = ds_selected["time"].values + import numpy as np + + time_min, time_max = np.min(time_values), np.max(time_values) + return ( + np.datetime64(schedule_start) >= time_min + and np.datetime64(schedule_end) <= time_max + ) + + if start_end_in_product_timerange( + selected_id, schedule_start, schedule_end, username, password + ): + return selected_id + else: + return ( + PRODUCT_IDS["phys"]["analysis"] + if physical + else BGC_ANALYSIS_IDS[variable] + ) + class Instrument(abc.ABC): """Base class for instruments and their simulation.""" @@ -93,21 +235,23 @@ def __init__( "time": "time", "depth": "depth", } # same dimensions for all instruments - self.bathymetry_file = self.directory.joinpath(bathymetry_file) + self.bathymetry_file = bathymetry_file self.add_bathymetry = add_bathymetry self.allow_time_extrapolation = allow_time_extrapolation def load_input_data(self) -> FieldSet: """Load and return the input data as a FieldSet for the instrument.""" # TODO: can simulate_schedule.py be refactored to be contained in base.py and repsective instrument files too...? - - # TODO: tests will need updating...! - - # TODO: think about combining InputDataset and Instrument classes together! Or maybe not if they are better kept separate... + # TODO: tests need updating...! try: + data_dir = self._get_data_dir(self.directory) + joined_filepaths = { + key: data_dir.joinpath(filename) + for key, filename in self.filenames.items() + } fieldset = FieldSet.from_netcdf( - self.filenames, + joined_filepaths, self.variables, self.dimensions, allow_time_extrapolation=self.allow_time_extrapolation, @@ -118,7 +262,7 @@ def load_input_data(self) -> FieldSet: ) from e # interpolation methods - for var in self.variables: + for var in (v for v in self.variables if v not in ("U", "V")): getattr(fieldset, var).interp_method = "linear_invdist_land_tracer" # depth negative for g in fieldset.gridset.grids: @@ -126,17 +270,18 @@ def load_input_data(self) -> FieldSet: # bathymetry data if self.add_bathymetry: bathymetry_field = Field.from_netcdf( - self.bathymetry_file, - self.bathymetry_variables, - self.bathymetry_dimensions, + data_dir.joinpath(self.bathymetry_file), + variable=("bathymetry", "deptho"), + dimensions={"lon": "longitude", "lat": "latitude"}, ) bathymetry_field.data = -bathymetry_field.data fieldset.add_field(bathymetry_field) fieldset.computeTimeChunk(0, 1) # read in data already + return fieldset @abc.abstractmethod - def simulate(self, measurements: list, out_path: str | Path): + def simulate(self, data_dir: Path, measurements: list, out_path: str | Path): """Simulate instrument measurements.""" def run(self, measurements: list, out_path: str | Path) -> None: @@ -150,3 +295,15 @@ def run(self, measurements: list, out_path: str | Path) -> None: ) as spinner: self.simulate(measurements, out_path) spinner.ok("✅") + + def _get_data_dir(self, expedition_dir: Path) -> Path: + space_time_region_hash = get_space_time_region_hash( + self.expedition.schedule.space_time_region + ) + data_dir = get_existing_download(expedition_dir, space_time_region_hash) + + assert data_dir is not None, ( + "Input data hasn't been found. Have you run the `virtualship fetch` command?" + ) + + return data_dir diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 82cace94a..c1981e062 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -8,7 +8,7 @@ from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.models import Spacetime -from virtualship.utils import register_input_dataset +from virtualship.utils import register_input_dataset, register_instrument @dataclass @@ -81,26 +81,27 @@ def get_datasets_dict(self) -> dict: """Get variable specific args for instrument.""" return { "Sdata": { - "dataset_id": "cmems_mod_glo_phy-so_anfc_0.083deg_PT6H-i", + "physical": True, "variables": ["so"], "output_filename": f"{self.name}_s.nc", }, "Tdata": { - "dataset_id": "cmems_mod_glo_phy-thetao_anfc_0.083deg_PT6H-i", + "physical": True, "variables": ["thetao"], "output_filename": f"{self.name}_t.nc", }, } +@register_instrument(InstrumentType.CTD) class CTDInstrument(Instrument): """CTD instrument class.""" - def __init__(self, name, expedition, directory): + def __init__(self, expedition, directory): """Initialize CTDInstrument.""" filenames = { - "S": directory.data_dir.joinpath(f"{name}_s.nc"), - "T": directory.data_dir.joinpath(f"{name}_t.nc"), + "S": f"{CTD.name}_s.nc", + "T": f"{CTD.name}_t.nc", } variables = {"S": "so", "T": "thetao"} @@ -114,13 +115,13 @@ def __init__(self, name, expedition, directory): allow_time_extrapolation=True, ) - def simulate(self) -> None: + def simulate(self, measurements, out_path) -> None: """Simulate CTD measurements.""" WINCH_SPEED = 1.0 # sink and rise speed in m/s DT = 10.0 # dt of CTD simulation integrator OUTPUT_DT = timedelta(seconds=10) # output dt for CTD simulation - if len(self.measurements) == 0: + if len(measurements) == 0: print( "No CTDs provided. Parcels currently crashes when providing an empty particle set, so no CTD simulation will be done and no files will be created." ) @@ -136,7 +137,7 @@ def simulate(self) -> None: if not all( [ np.datetime64(ctd.spacetime.time) >= fieldset_starttime - for ctd in self.measurements + for ctd in measurements ] ): raise ValueError("CTD deployed before fieldset starts.") @@ -152,7 +153,7 @@ def simulate(self) -> None: time=0, ), ) - for ctd in self.measurements + for ctd in measurements ] # CTD depth can not be too shallow, because kernel would break. @@ -176,7 +177,7 @@ def simulate(self) -> None: ) # define output file for the simulation - out_file = ctd_particleset.ParticleFile(name=self.out_path, outputdt=OUTPUT_DT) + out_file = ctd_particleset.ParticleFile(name=out_path, outputdt=OUTPUT_DT) # execute simulation ctd_particleset.execute( diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index b017e0f4e..ed9ab9d94 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -106,42 +106,37 @@ def get_datasets_dict(self) -> dict: """Variable specific args for instrument.""" return { "o2data": { - "dataset_id": "cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m", + "physical": False, "variables": ["o2"], "output_filename": f"{self.name}_o2.nc", }, "chlorodata": { - "dataset_id": "cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m", + "physical": False, "variables": ["chl"], "output_filename": f"{self.name}_chl.nc", }, "nitratedata": { - "dataset_id": "cmems_mod_glo_bgc-nut_anfc_0.25deg_P1D-m", + "physical": False, "variables": ["no3"], "output_filename": f"{self.name}_no3.nc", }, "phosphatedata": { - "dataset_id": "cmems_mod_glo_bgc-nut_anfc_0.25deg_P1D-m", + "physical": False, "variables": ["po4"], "output_filename": f"{self.name}_po4.nc", }, "phdata": { - "dataset_id": "cmems_mod_glo_bgc-car_anfc_0.25deg_P1D-m", + "physical": False, "variables": ["ph"], "output_filename": f"{self.name}_ph.nc", }, "phytoplanktondata": { - "dataset_id": "cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m", + "physical": False, "variables": ["phyc"], "output_filename": f"{self.name}_phyc.nc", }, - "zooplanktondata": { - "dataset_id": "cmems_mod_glo_bgc-plankton_anfc_0.25deg_P1D-m", - "variables": ["zooc"], - "output_filename": f"{self.name}_zooc.nc", - }, "primaryproductiondata": { - "dataset_id": "cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m", + "physical": False, "variables": ["nppv"], "output_filename": f"{self.name}_nppv.nc", }, @@ -152,17 +147,16 @@ def get_datasets_dict(self) -> dict: class CTD_BGCInstrument(Instrument): """CTD_BGC instrument class.""" - def __init__(self, name, expedition, directory): + def __init__(self, expedition, directory): """Initialize CTD_BGCInstrument.""" filenames = { - "o2": directory.joinpath(f"{name}_o2.nc"), - "chl": directory.joinpath(f"{name}_chl.nc"), - "no3": directory.joinpath(f"{name}_no3.nc"), - "po4": directory.joinpath(f"{name}_po4.nc"), - "ph": directory.joinpath(f"{name}_ph.nc"), - "phyc": directory.joinpath(f"{name}_phyc.nc"), - "zooc": directory.joinpath(f"{name}_zooc.nc"), - "nppv": directory.joinpath(f"{name}_nppv.nc"), + "o2": f"{CTD_BGC.name}_o2.nc", + "chl": f"{CTD_BGC.name}_chl.nc", + "no3": f"{CTD_BGC.name}_no3.nc", + "po4": f"{CTD_BGC.name}_po4.nc", + "ph": f"{CTD_BGC.name}_ph.nc", + "phyc": f"{CTD_BGC.name}_phyc.nc", + "nppv": f"{CTD_BGC.name}_nppv.nc", } variables = { "o2": "o2", @@ -171,7 +165,6 @@ def __init__(self, name, expedition, directory): "po4": "po4", "ph": "ph", "phyc": "phyc", - "zooc": "zooc", "nppv": "nppv", } super().__init__( @@ -184,13 +177,13 @@ def __init__(self, name, expedition, directory): allow_time_extrapolation=True, ) - def simulate(self) -> None: + def simulate(self, measurements, out_path) -> None: """Simulate BGC CTD measurements using Parcels.""" WINCH_SPEED = 1.0 # sink and rise speed in m/s DT = 10.0 # dt of CTD_BGC simulation integrator OUTPUT_DT = timedelta(seconds=10) # output dt for CTD_BGC simulation - if len(self.measurements) == 0: + if len(measurements) == 0: print( "No BGC CTDs provided. Parcels currently crashes when providing an empty particle set, so no BGC CTD simulation will be done and no files will be created." ) @@ -206,7 +199,7 @@ def simulate(self) -> None: if not all( [ np.datetime64(ctd_bgc.spacetime.time) >= fieldset_starttime - for ctd_bgc in self.measurements + for ctd_bgc in measurements ] ): raise ValueError("BGC CTD deployed before fieldset starts.") @@ -246,9 +239,7 @@ def simulate(self) -> None: ) # define output file for the simulation - out_file = ctd_bgc_particleset.ParticleFile( - name=self.out_path, outputdt=OUTPUT_DT - ) + out_file = ctd_bgc_particleset.ParticleFile(name=out_path, outputdt=OUTPUT_DT) # execute simulation ctd_bgc_particleset.execute( diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index 72db065ed..93cd26310 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -70,12 +70,12 @@ def get_datasets_dict(self) -> dict: """Get variable specific args for instrument.""" return { "UVdata": { - "dataset_id": "cmems_mod_glo_phy-cur_anfc_0.083deg_PT6H-i", + "physical": True, "variables": ["uo", "vo"], "output_filename": f"{self.name}_uv.nc", }, "Tdata": { - "dataset_id": "cmems_mod_glo_phy-thetao_anfc_0.083deg_PT6H-i", + "physical": True, "variables": ["thetao"], "output_filename": f"{self.name}_t.nc", }, @@ -86,13 +86,14 @@ def get_datasets_dict(self) -> dict: class DrifterInstrument(Instrument): """Drifter instrument class.""" - def __init__(self, name, expedition, directory): + def __init__(self, expedition, directory): """Initialize DrifterInstrument.""" filenames = { - "UV": directory.joinpath(f"{name}_uv.nc"), - "T": directory.joinpath(f"{name}_t.nc"), + "U": f"{Drifter.name}_uv.nc", + "V": f"{Drifter.name}_uv.nc", + "T": f"{Drifter.name}_t.nc", } - variables = {"UV": ["uo", "vo"], "T": "thetao"} + variables = {"U": "uo", "V": "vo", "T": "thetao"} super().__init__( Drifter.name, expedition, @@ -103,13 +104,13 @@ def __init__(self, name, expedition, directory): allow_time_extrapolation=False, ) - def simulate(self) -> None: + def simulate(self, measurements, out_path) -> None: """Simulate Drifter measurements.""" OUTPUT_DT = timedelta(hours=5) DT = timedelta(minutes=5) ENDTIME = None - if len(self.measurements) == 0: + if len(measurements) == 0: print( "No drifters provided. Parcels currently crashes when providing an empty particle set, so no drifter simulation will be done and no files will be created." ) @@ -122,23 +123,22 @@ def simulate(self) -> None: drifter_particleset = ParticleSet( fieldset=fieldset, pclass=_DrifterParticle, - lat=[drifter.spacetime.location.lat for drifter in self.measurements], - lon=[drifter.spacetime.location.lon for drifter in self.measurements], - depth=[drifter.depth for drifter in self.measurements], - time=[drifter.spacetime.time for drifter in self.measurements], + lat=[drifter.spacetime.location.lat for drifter in measurements], + lon=[drifter.spacetime.location.lon for drifter in measurements], + depth=[drifter.depth for drifter in measurements], + time=[drifter.spacetime.time for drifter in measurements], has_lifetime=[ - 1 if drifter.lifetime is not None else 0 - for drifter in self.measurements + 1 if drifter.lifetime is not None else 0 for drifter in measurements ], lifetime=[ 0 if drifter.lifetime is None else drifter.lifetime.total_seconds() - for drifter in self.measurements + for drifter in measurements ], ) # define output file for the simulation out_file = drifter_particleset.ParticleFile( - name=self.out_path, + name=out_path, outputdt=OUTPUT_DT, chunks=[len(drifter_particleset), 100], ) diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index 9c0fc4011..4160f474a 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -62,12 +62,12 @@ def get_datasets_dict(self) -> dict: """Get variable specific args for instrument.""" return { "Sdata": { - "dataset_id": "cmems_mod_glo_phy-so_anfc_0.083deg_PT6H-i", + "physical": True, "variables": ["so"], "output_filename": f"{self.name}_s.nc", }, "Tdata": { - "dataset_id": "cmems_mod_glo_phy-thetao_anfc_0.083deg_PT6H-i", + "physical": True, "variables": ["thetao"], "output_filename": f"{self.name}_t.nc", }, @@ -78,11 +78,11 @@ def get_datasets_dict(self) -> dict: class Underwater_STInstrument(Instrument): """Underwater_ST instrument class.""" - def __init__(self, name, expedition, directory): + def __init__(self, expedition, directory): """Initialize Underwater_STInstrument.""" filenames = { - "S": directory.joinpath(f"{name}_s.nc"), - "T": directory.joinpath(f"{name}_t.nc"), + "S": f"{Underwater_ST.name}_s.nc", + "T": f"{Underwater_ST.name}_t.nc", } variables = {"S": "so", "T": "thetao"} super().__init__( @@ -95,14 +95,13 @@ def __init__(self, name, expedition, directory): allow_time_extrapolation=True, ) - def simulate(self) -> None: + def simulate(self, measurements, out_path) -> None: """Simulate underway salinity and temperature measurements.""" DEPTH = -2.0 - self.measurements.sort(key=lambda p: p.time) + measurements.sort(key=lambda p: p.time) fieldset = self.load_input_data() - particleset = ParticleSet.from_list( fieldset=fieldset, pclass=_ShipSTParticle, @@ -112,9 +111,9 @@ def simulate(self) -> None: time=0, ) - out_file = particleset.ParticleFile(name=self.out_path, outputdt=np.inf) + out_file = particleset.ParticleFile(name=out_path, outputdt=np.inf) - for point in self.measurements: + for point in measurements: particleset.lon_nextloop[:] = point.location.lon particleset.lat_nextloop[:] = point.location.lat particleset.time_nextloop[:] = fieldset.time_origin.reltime( diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index 7c916d98e..918ef9a20 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -84,17 +84,17 @@ def get_datasets_dict(self) -> dict: """Get variable specific args for instrument.""" return { "UVdata": { - "dataset_id": "cmems_mod_glo_phy-cur_anfc_0.083deg_PT6H-i", + "physical": True, "variables": ["uo", "vo"], "output_filename": f"{self.name}_uv.nc", }, "Sdata": { - "dataset_id": "cmems_mod_glo_phy-so_anfc_0.083deg_PT6H-i", + "physical": True, "variables": ["so"], "output_filename": f"{self.name}_s.nc", }, "Tdata": { - "dataset_id": "cmems_mod_glo_phy-thetao_anfc_0.083deg_PT6H-i", + "physical": True, "variables": ["thetao"], "output_filename": f"{self.name}_t.nc", }, @@ -105,14 +105,15 @@ def get_datasets_dict(self) -> dict: class XBTInstrument(Instrument): """XBT instrument class.""" - def __init__(self, name, expedition, directory): + def __init__(self, expedition, directory): """Initialize XBTInstrument.""" filenames = { - "UV": directory.joinpath(f"{name}_uv.nc"), - "S": directory.joinpath(f"{name}_s.nc"), - "T": directory.joinpath(f"{name}_t.nc"), + "U": f"{XBT.name}_uv.nc", + "V": f"{XBT.name}_uv.nc", + "S": f"{XBT.name}_s.nc", + "T": f"{XBT.name}_t.nc", } - variables = {"UV": ["uo", "vo"], "S": "so", "T": "thetao"} + variables = {"U": "uo", "V": "vo", "S": "so", "T": "thetao"} super().__init__( XBT.name, expedition, @@ -123,12 +124,12 @@ def __init__(self, name, expedition, directory): allow_time_extrapolation=True, ) - def simulate(self) -> None: + def simulate(self, measurements, out_path) -> None: """Simulate XBT measurements.""" DT = 10.0 # dt of XBT simulation integrator OUTPUT_DT = timedelta(seconds=1) - if len(self.measurements) == 0: + if len(measurements) == 0: print( "No XBTs provided. Parcels currently crashes when providing an empty particle set, so no XBT simulation will be done and no files will be created." ) @@ -144,7 +145,7 @@ def simulate(self) -> None: if not all( [ np.datetime64(xbt.spacetime.time) >= fieldset_starttime - for xbt in self.measurements + for xbt in measurements ] ): raise ValueError("XBT deployed before fieldset starts.") @@ -160,7 +161,7 @@ def simulate(self) -> None: time=0, ), ) - for xbt in self.measurements + for xbt in measurements ] # initial fall speeds @@ -186,7 +187,7 @@ def simulate(self) -> None: fall_speed=[xbt.fall_speed for xbt in self.measurements], ) - out_file = xbt_particleset.ParticleFile(name=self.out_path, outputdt=OUTPUT_DT) + out_file = xbt_particleset.ParticleFile(name=out_path, outputdt=OUTPUT_DT) xbt_particleset.execute( [_sample_temperature, _xbt_cast], diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index 8888f6572..c70de855b 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -62,7 +62,7 @@ def get_instruments(self) -> set[InstrumentType]: instruments_in_expedition.append(InstrumentType.ADCP) if self.instruments_config.ship_underwater_st_config is not None: instruments_in_expedition.append(InstrumentType.UNDERWATER_ST) - return set(instruments_in_expedition) + return sorted(set(instruments_in_expedition), key=lambda x: x.name) except Exception as e: raise InstrumentsConfigError( "Underway instrument config attribute(s) are missing from YAML. Must be Config object or None." @@ -90,7 +90,8 @@ class Schedule(pydantic.BaseModel): def verify( self, ship_speed: float, - input_dir: str | Path | None, + data_dir: str | Path | None, + ignore_missing_bathymetry: bool = False, *, check_space_time_region: bool = False, ) -> None: @@ -103,14 +104,6 @@ def verify( 3. Waypoint times are in ascending order. 4. All waypoints are in water (not on land). 5. The ship can arrive on time at each waypoint given its speed. - - :param ship_speed: The ship's speed in knots. - :param input_dir: The input directory containing necessary files. - :param check_space_time_region: whether to check for missing space_time_region. - # :param ignore_missing_fieldsets: whether to ignore warning for missing field sets. - :raises PlanningError: If any of the verification checks fail, indicating infeasible or incorrect waypoints. - :raises NotImplementedError: If an instrument in the schedule is not implemented. - :return: None. The method doesn't return a value but raises exceptions if verification fails. """ print("\nVerifying route... ") @@ -139,18 +132,20 @@ def verify( # check if all waypoints are in water using bathymetry data # TODO: tests should be updated to check this! + # TODO: write test that checks that will flag when waypoint is on land!! [add to existing suite of fail .verify() tests in test_expedition.py] + # TODO: need to do an overhaul of the DATA which is in tests/expedition/expedition_dir - don't think it's currently suitable! land_waypoints = [] - if input_dir is not None: - bathymetry_path = input_dir.joinpath("bathymetry.nc") + if data_dir is not None: + bathymetry_path = data_dir.joinpath("bathymetry.nc") try: bathymetry_field = Field.from_netcdf( bathymetry_path, - variables=("bathymetry", "deptho"), + variable=("bathymetry", "deptho"), dimensions={"lon": "longitude", "lat": "latitude"}, ) except Exception as e: - raise FileNotFoundError( - "Bathymetry file not found in input data. Cannot verify waypoints are in water." + raise ScheduleError( + f"Problem loading bathymetry data (used to verify waypoints are in water): {e}" ) from e for wp_i, wp in enumerate(self.waypoints): bathy = bathymetry_field.eval( @@ -159,12 +154,17 @@ def verify( wp.location.lat, wp.location.lon, ) - if np.isnan(bathy) or bathy >= 0: + if np.isnan(bathy) or bathy <= 0: land_waypoints.append((wp_i, wp)) + if len(land_waypoints) > 0: raise ScheduleError( f"The following waypoints are on land: {['#' + str(wp_i + 1) + ' ' + str(wp) for (wp_i, wp) in land_waypoints]}" ) + elif not ignore_missing_bathymetry: + raise ScheduleError( + "Cannot verify waypoints are in water as bathymetry data not found. Have you run `virtualship fetch` command?" + ) # check that ship will arrive on time at each waypoint (in case no unexpected event happen) time = self.waypoints[0].time From 1081fab71eb481d665a8f66c1d8dea3c09b72d53 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 29 Oct 2025 13:01:08 +0100 Subject: [PATCH 33/97] adding U and V to instruments where missing --- src/virtualship/instruments/base.py | 19 ++++++------ src/virtualship/instruments/ctd.py | 29 +++++++++++++------ src/virtualship/instruments/ctd_bgc.py | 17 +++++++++-- .../instruments/ship_underwater_st.py | 10 ++++++- 4 files changed, 54 insertions(+), 21 deletions(-) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 6ae38cc0f..c45c71fec 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -4,13 +4,11 @@ import copernicusmarine import numpy as np -from yaspin import yaspin from parcels import Field, FieldSet from virtualship.cli._fetch import get_existing_download, get_space_time_region_hash from virtualship.errors import CopernicusCatalogueError from virtualship.models import Expedition, SpaceTimeRegion -from virtualship.utils import ship_spinner PRODUCT_IDS = { "phys": { @@ -48,6 +46,9 @@ class InputDataset(abc.ABC): """Base class for instrument input datasets.""" + # TODO: data download is performed per instrument (in `fetch`), which is a bit inefficient when some instruments can share dataa. + # TODO: However, future changes, with Parcels-v4 and copernicusmarine direct ingestion, will hopefully remove the need for fetch. + def __init__( self, name: str, @@ -288,13 +289,13 @@ def run(self, measurements: list, out_path: str | Path) -> None: """Run instrument simulation.""" # TODO: this will have to be able to handle the non-spinner/instead progress bar for drifters and argos! - with yaspin( - text=f"Simulating {self.name} measurements... ", - side="right", - spinner=ship_spinner, - ) as spinner: - self.simulate(measurements, out_path) - spinner.ok("✅") + # with yaspin( + # text=f"Simulating {self.name} measurements... ", + # side="right", + # spinner=ship_spinner, + # ) as spinner: + self.simulate(measurements, out_path) + # spinner.ok("✅") def _get_data_dir(self, expedition_dir: Path) -> Path: space_time_region_hash = get_space_time_region_hash( diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index c1981e062..ffc88922e 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -80,6 +80,11 @@ def __init__(self, data_dir, credentials, space_time_region): def get_datasets_dict(self) -> dict: """Get variable specific args for instrument.""" return { + "UVdata": { + "physical": True, # TODO: U and V are only needed for parcels.FieldSet.check_complete()... would be nice to remove... v4? + "variables": ["uo", "vo"], + "output_filename": f"{self.name}_uv.nc", + }, "Sdata": { "physical": True, "variables": ["so"], @@ -100,10 +105,12 @@ class CTDInstrument(Instrument): def __init__(self, expedition, directory): """Initialize CTDInstrument.""" filenames = { + "U": f"{CTD.name}_uv.nc", # TODO: U and V are only needed for parcels.FieldSet.check_complete()... would be nice to remove... v4? + "V": f"{CTD.name}_uv.nc", "S": f"{CTD.name}_s.nc", "T": f"{CTD.name}_t.nc", } - variables = {"S": "so", "T": "thetao"} + variables = {"U": "uo", "V": "vo", "S": "so", "T": "thetao"} super().__init__( CTD.name, @@ -130,8 +137,12 @@ def simulate(self, measurements, out_path) -> None: fieldset = self.load_input_data() - fieldset_starttime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[0]) - fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) + fieldset_starttime = fieldset.T.grid.time_origin.fulltime( + fieldset.T.grid.time_full[0] + ) + fieldset_endtime = fieldset.T.grid.time_origin.fulltime( + fieldset.T.grid.time_full[-1] + ) # deploy time for all ctds should be later than fieldset start time if not all( @@ -167,13 +178,13 @@ def simulate(self, measurements, out_path) -> None: ctd_particleset = ParticleSet( fieldset=fieldset, pclass=_CTDParticle, - lon=[ctd.spacetime.location.lon for ctd in self.measurements], - lat=[ctd.spacetime.location.lat for ctd in self.measurements], - depth=[ctd.min_depth for ctd in self.measurements], - time=[ctd.spacetime.time for ctd in self.measurements], + lon=[ctd.spacetime.location.lon for ctd in measurements], + lat=[ctd.spacetime.location.lat for ctd in measurements], + depth=[ctd.min_depth for ctd in measurements], + time=[ctd.spacetime.time for ctd in measurements], max_depth=max_depths, - min_depth=[ctd.min_depth for ctd in self.measurements], - winch_speed=[WINCH_SPEED for _ in self.measurements], + min_depth=[ctd.min_depth for ctd in measurements], + winch_speed=[WINCH_SPEED for _ in measurements], ) # define output file for the simulation diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index ed9ab9d94..1b702b6ea 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -105,6 +105,11 @@ def __init__(self, data_dir, credentials, space_time_region): def get_datasets_dict(self) -> dict: """Variable specific args for instrument.""" return { + "UVdata": { + "physical": True, # TODO: U and V are only needed for parcels.FieldSet.check_complete()... would be nice to remove... v4? + "variables": ["uo", "vo"], + "output_filename": f"{self.name}_uv.nc", + }, "o2data": { "physical": False, "variables": ["o2"], @@ -150,6 +155,8 @@ class CTD_BGCInstrument(Instrument): def __init__(self, expedition, directory): """Initialize CTD_BGCInstrument.""" filenames = { + "U": f"{CTD_BGC.name}_uv.nc", # TODO: U and V are only needed for parcels.FieldSet.check_complete()... would be nice to remove... v4? + "V": f"{CTD_BGC.name}_uv.nc", "o2": f"{CTD_BGC.name}_o2.nc", "chl": f"{CTD_BGC.name}_chl.nc", "no3": f"{CTD_BGC.name}_no3.nc", @@ -159,6 +166,8 @@ def __init__(self, expedition, directory): "nppv": f"{CTD_BGC.name}_nppv.nc", } variables = { + "U": "uo", + "V": "vo", "o2": "o2", "chl": "chl", "no3": "no3", @@ -192,8 +201,12 @@ def simulate(self, measurements, out_path) -> None: fieldset = self.load_input_data() - fieldset_starttime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[0]) - fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) + fieldset_starttime = fieldset.o2.grid.time_origin.fulltime( + fieldset.o2.grid.time_full[0] + ) + fieldset_endtime = fieldset.o2.grid.time_origin.fulltime( + fieldset.o2.grid.time_full[-1] + ) # deploy time for all ctds should be later than fieldset start time if not all( diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index 4160f474a..2ffbd7626 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -61,6 +61,11 @@ def __init__(self, data_dir, credentials, space_time_region): def get_datasets_dict(self) -> dict: """Get variable specific args for instrument.""" return { + "UVdata": { + "physical": True, # TODO: U and V are only needed for parcels.FieldSet.check_complete()... would be nice to remove... v4? + "variables": ["uo", "vo"], + "output_filename": f"{self.name}_uv.nc", + }, "Sdata": { "physical": True, "variables": ["so"], @@ -81,10 +86,13 @@ class Underwater_STInstrument(Instrument): def __init__(self, expedition, directory): """Initialize Underwater_STInstrument.""" filenames = { + "U": f"{Underwater_ST.name}_uv.nc", # TODO: U and V are only needed for parcels.FieldSet.check_complete()... would be nice to remove... v4? + "V": f"{Underwater_ST.name}_uv.nc", "S": f"{Underwater_ST.name}_s.nc", "T": f"{Underwater_ST.name}_t.nc", } - variables = {"S": "so", "T": "thetao"} + variables = {"U": "uo", "V": "vo", "S": "so", "T": "thetao"} + super().__init__( Underwater_ST.name, expedition, From a0d7d2c64d033c2ac71c171a58c59aba3c61a4fb Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 29 Oct 2025 13:07:43 +0100 Subject: [PATCH 34/97] enhanced error messaging for XBT in too shallow regions --- src/virtualship/instruments/xbt.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index 918ef9a20..b15c0e6fd 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -165,26 +165,26 @@ def simulate(self, measurements, out_path) -> None: ] # initial fall speeds - initial_fall_speeds = [xbt.fall_speed for xbt in self.measurements] + initial_fall_speeds = [xbt.fall_speed for xbt in measurements] # XBT depth can not be too shallow, because kernel would break. for max_depth, fall_speed in zip(max_depths, initial_fall_speeds, strict=False): if not max_depth <= -DT * fall_speed: raise ValueError( - f"XBT max_depth or bathymetry shallower than maximum {-DT * fall_speed}" + f"XBT max_depth or bathymetry shallower than minimum {-DT * fall_speed}. It is likely the XBT cannot be deployed in this area, which is too shallow." ) # define xbt particles xbt_particleset = ParticleSet( fieldset=fieldset, pclass=_XBTParticle, - lon=[xbt.spacetime.location.lon for xbt in self.measurements], - lat=[xbt.spacetime.location.lat for xbt in self.measurements], - depth=[xbt.min_depth for xbt in self.measurements], - time=[xbt.spacetime.time for xbt in self.measurements], + lon=[xbt.spacetime.location.lon for xbt in measurements], + lat=[xbt.spacetime.location.lat for xbt in measurements], + depth=[xbt.min_depth for xbt in measurements], + time=[xbt.spacetime.time for xbt in measurements], max_depth=max_depths, - min_depth=[xbt.min_depth for xbt in self.measurements], - fall_speed=[xbt.fall_speed for xbt in self.measurements], + min_depth=[xbt.min_depth for xbt in measurements], + fall_speed=[xbt.fall_speed for xbt in measurements], ) out_file = xbt_particleset.ParticleFile(name=out_path, outputdt=OUTPUT_DT) From e5c08ce049be8440d7039cb2eb2e9af7f79ba59a Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 29 Oct 2025 13:07:55 +0100 Subject: [PATCH 35/97] bug fixes --- src/virtualship/instruments/ctd_bgc.py | 14 +++++++------- src/virtualship/instruments/drifter.py | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 1b702b6ea..f3174fd90 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -228,7 +228,7 @@ def simulate(self, measurements, out_path) -> None: time=0, ), ) - for ctd_bgc in self.measurements + for ctd_bgc in measurements ] # CTD depth can not be too shallow, because kernel would break. @@ -242,13 +242,13 @@ def simulate(self, measurements, out_path) -> None: ctd_bgc_particleset = ParticleSet( fieldset=fieldset, pclass=_CTD_BGCParticle, - lon=[ctd_bgc.spacetime.location.lon for ctd_bgc in self.measurements], - lat=[ctd_bgc.spacetime.location.lat for ctd_bgc in self.measurements], - depth=[ctd_bgc.min_depth for ctd_bgc in self.measurements], - time=[ctd_bgc.spacetime.time for ctd_bgc in self.measurements], + lon=[ctd_bgc.spacetime.location.lon for ctd_bgc in measurements], + lat=[ctd_bgc.spacetime.location.lat for ctd_bgc in measurements], + depth=[ctd_bgc.min_depth for ctd_bgc in measurements], + time=[ctd_bgc.spacetime.time for ctd_bgc in measurements], max_depth=max_depths, - min_depth=[ctd_bgc.min_depth for ctd_bgc in self.measurements], - winch_speed=[WINCH_SPEED for _ in self.measurements], + min_depth=[ctd_bgc.min_depth for ctd_bgc in measurements], + winch_speed=[WINCH_SPEED for _ in measurements], ) # define output file for the simulation diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index 93cd26310..f33fde553 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -164,7 +164,7 @@ def simulate(self, measurements, out_path) -> None: # if there are more particles left than the number of drifters with an indefinite endtime, warn the user if len(drifter_particleset.particledata) > len( - [d for d in self.measurements if d.lifetime is None] + [d for d in measurements if d.lifetime is None] ): print( "WARN: Some drifters had a life time beyond the end time of the fieldset or the requested end time." From a4f8af0c88f65837eccd82fd420c7904cd8962f3 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 3 Nov 2025 08:43:52 +0100 Subject: [PATCH 36/97] version U and V downloaded --- src/virtualship/instruments/base.py | 2 ++ src/virtualship/instruments/ctd.py | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index c45c71fec..23b324e58 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -245,6 +245,8 @@ def load_input_data(self) -> FieldSet: # TODO: can simulate_schedule.py be refactored to be contained in base.py and repsective instrument files too...? # TODO: tests need updating...! + #! TODO: CTD, CTD_BGC and Underway_ST deployment testing in `run` is outstanding! + try: data_dir = self._get_data_dir(self.directory) joined_filepaths = { diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index ffc88922e..86c6e0aca 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -10,6 +10,9 @@ from virtualship.models import Spacetime from virtualship.utils import register_input_dataset, register_instrument +# TODO: add some kind of check that each instrument has a dataclass, particle class, InputDataset class and Instrument class? +# TODO: probably as a test + @dataclass class CTD: @@ -33,6 +36,9 @@ class CTD: ) +# TODO: way to group kernels together, just to make clearer? + + def _sample_temperature(particle, fieldset, time): particle.temperature = fieldset.T[time, particle.depth, particle.lat, particle.lon] @@ -104,6 +110,8 @@ class CTDInstrument(Instrument): def __init__(self, expedition, directory): """Initialize CTDInstrument.""" + #! TODO: actually don't need to download U and V for CTD simulation... can instead add mock/duplicate of T and name it U (also don't need V)! + filenames = { "U": f"{CTD.name}_uv.nc", # TODO: U and V are only needed for parcels.FieldSet.check_complete()... would be nice to remove... v4? "V": f"{CTD.name}_uv.nc", From e3c57f628586b89344734b276ca121f668b1fcdd Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 3 Nov 2025 10:30:09 +0100 Subject: [PATCH 37/97] dummy U and V --- src/virtualship/instruments/base.py | 19 +++++++------- src/virtualship/instruments/ctd.py | 14 ++++------- src/virtualship/instruments/ctd_bgc.py | 14 +++-------- .../instruments/ship_underwater_st.py | 15 +++++------ src/virtualship/utils.py | 25 +++++++++++++++++++ 5 files changed, 49 insertions(+), 38 deletions(-) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 23b324e58..6c036b288 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -4,11 +4,13 @@ import copernicusmarine import numpy as np +from yaspin import yaspin from parcels import Field, FieldSet from virtualship.cli._fetch import get_existing_download, get_space_time_region_hash from virtualship.errors import CopernicusCatalogueError from virtualship.models import Expedition, SpaceTimeRegion +from virtualship.utils import ship_spinner PRODUCT_IDS = { "phys": { @@ -244,9 +246,6 @@ def load_input_data(self) -> FieldSet: """Load and return the input data as a FieldSet for the instrument.""" # TODO: can simulate_schedule.py be refactored to be contained in base.py and repsective instrument files too...? # TODO: tests need updating...! - - #! TODO: CTD, CTD_BGC and Underway_ST deployment testing in `run` is outstanding! - try: data_dir = self._get_data_dir(self.directory) joined_filepaths = { @@ -291,13 +290,13 @@ def run(self, measurements: list, out_path: str | Path) -> None: """Run instrument simulation.""" # TODO: this will have to be able to handle the non-spinner/instead progress bar for drifters and argos! - # with yaspin( - # text=f"Simulating {self.name} measurements... ", - # side="right", - # spinner=ship_spinner, - # ) as spinner: - self.simulate(measurements, out_path) - # spinner.ok("✅") + with yaspin( + text=f"Simulating {self.name} measurements... ", + side="right", + spinner=ship_spinner, + ) as spinner: + self.simulate(measurements, out_path) + spinner.ok("✅") def _get_data_dir(self, expedition_dir: Path) -> Path: space_time_region_hash = get_space_time_region_hash( diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 86c6e0aca..22c69799f 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -8,7 +8,7 @@ from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.models import Spacetime -from virtualship.utils import register_input_dataset, register_instrument +from virtualship.utils import add_dummy_UV, register_input_dataset, register_instrument # TODO: add some kind of check that each instrument has a dataclass, particle class, InputDataset class and Instrument class? # TODO: probably as a test @@ -86,11 +86,6 @@ def __init__(self, data_dir, credentials, space_time_region): def get_datasets_dict(self) -> dict: """Get variable specific args for instrument.""" return { - "UVdata": { - "physical": True, # TODO: U and V are only needed for parcels.FieldSet.check_complete()... would be nice to remove... v4? - "variables": ["uo", "vo"], - "output_filename": f"{self.name}_uv.nc", - }, "Sdata": { "physical": True, "variables": ["so"], @@ -113,12 +108,10 @@ def __init__(self, expedition, directory): #! TODO: actually don't need to download U and V for CTD simulation... can instead add mock/duplicate of T and name it U (also don't need V)! filenames = { - "U": f"{CTD.name}_uv.nc", # TODO: U and V are only needed for parcels.FieldSet.check_complete()... would be nice to remove... v4? - "V": f"{CTD.name}_uv.nc", "S": f"{CTD.name}_s.nc", "T": f"{CTD.name}_t.nc", } - variables = {"U": "uo", "V": "vo", "S": "so", "T": "thetao"} + variables = {"S": "so", "T": "thetao"} super().__init__( CTD.name, @@ -145,6 +138,9 @@ def simulate(self, measurements, out_path) -> None: fieldset = self.load_input_data() + # add dummy U + add_dummy_UV(fieldset) # TODO: parcels v3 bodge; remove when parcels v4 is used + fieldset_starttime = fieldset.T.grid.time_origin.fulltime( fieldset.T.grid.time_full[0] ) diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index f3174fd90..c42458234 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -8,7 +8,7 @@ from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime -from virtualship.utils import register_input_dataset, register_instrument +from virtualship.utils import add_dummy_UV, register_input_dataset, register_instrument @dataclass @@ -105,11 +105,6 @@ def __init__(self, data_dir, credentials, space_time_region): def get_datasets_dict(self) -> dict: """Variable specific args for instrument.""" return { - "UVdata": { - "physical": True, # TODO: U and V are only needed for parcels.FieldSet.check_complete()... would be nice to remove... v4? - "variables": ["uo", "vo"], - "output_filename": f"{self.name}_uv.nc", - }, "o2data": { "physical": False, "variables": ["o2"], @@ -155,8 +150,6 @@ class CTD_BGCInstrument(Instrument): def __init__(self, expedition, directory): """Initialize CTD_BGCInstrument.""" filenames = { - "U": f"{CTD_BGC.name}_uv.nc", # TODO: U and V are only needed for parcels.FieldSet.check_complete()... would be nice to remove... v4? - "V": f"{CTD_BGC.name}_uv.nc", "o2": f"{CTD_BGC.name}_o2.nc", "chl": f"{CTD_BGC.name}_chl.nc", "no3": f"{CTD_BGC.name}_no3.nc", @@ -166,8 +159,6 @@ def __init__(self, expedition, directory): "nppv": f"{CTD_BGC.name}_nppv.nc", } variables = { - "U": "uo", - "V": "vo", "o2": "o2", "chl": "chl", "no3": "no3", @@ -201,6 +192,9 @@ def simulate(self, measurements, out_path) -> None: fieldset = self.load_input_data() + # add dummy U + add_dummy_UV(fieldset) # TODO: parcels v3 bodge; remove when parcels v4 is used + fieldset_starttime = fieldset.o2.grid.time_origin.fulltime( fieldset.o2.grid.time_full[0] ) diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index 2ffbd7626..f23937784 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -6,7 +6,7 @@ from parcels import ParticleSet, ScipyParticle, Variable from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType -from virtualship.utils import register_input_dataset, register_instrument +from virtualship.utils import add_dummy_UV, register_input_dataset, register_instrument @dataclass @@ -61,11 +61,6 @@ def __init__(self, data_dir, credentials, space_time_region): def get_datasets_dict(self) -> dict: """Get variable specific args for instrument.""" return { - "UVdata": { - "physical": True, # TODO: U and V are only needed for parcels.FieldSet.check_complete()... would be nice to remove... v4? - "variables": ["uo", "vo"], - "output_filename": f"{self.name}_uv.nc", - }, "Sdata": { "physical": True, "variables": ["so"], @@ -86,12 +81,10 @@ class Underwater_STInstrument(Instrument): def __init__(self, expedition, directory): """Initialize Underwater_STInstrument.""" filenames = { - "U": f"{Underwater_ST.name}_uv.nc", # TODO: U and V are only needed for parcels.FieldSet.check_complete()... would be nice to remove... v4? - "V": f"{Underwater_ST.name}_uv.nc", "S": f"{Underwater_ST.name}_s.nc", "T": f"{Underwater_ST.name}_t.nc", } - variables = {"U": "uo", "V": "vo", "S": "so", "T": "thetao"} + variables = {"S": "so", "T": "thetao"} super().__init__( Underwater_ST.name, @@ -110,6 +103,10 @@ def simulate(self, measurements, out_path) -> None: measurements.sort(key=lambda p: p.time) fieldset = self.load_input_data() + + # add dummy U + add_dummy_UV(fieldset) # TODO: parcels v3 bodge; remove when parcels v4 is used + particleset = ParticleSet.from_list( fieldset=fieldset, pclass=_ShipSTParticle, diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index 6ffaefe73..324f8c672 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -8,6 +8,8 @@ from pathlib import Path from typing import TYPE_CHECKING, TextIO +from parcels import FieldSet + if TYPE_CHECKING: from virtualship.models import Expedition @@ -268,3 +270,26 @@ def get_input_dataset_class(instrument_type): def get_instrument_class(instrument_type): return INSTRUMENT_CLASS_MAP.get(instrument_type) + + +def add_dummy_UV(fieldset: FieldSet): + """Add a dummy U and V field to a FieldSet to satisfy parcels FieldSet completeness checks.""" + if "U" not in fieldset.__dict__.keys(): + for uv_var in ["U", "V"]: + dummy_field = getattr( + FieldSet.from_data( + {"U": 0, "V": 0}, {"lon": 0, "lat": 0}, mesh="spherical" + ), + uv_var, + ) + fieldset.add_field(dummy_field) + try: + fieldset.time_origin = ( + fieldset.T.grid.time_origin + if "T" in fieldset.__dict__.keys() + else fieldset.o2.grid.time_origin + ) + except Exception: + raise ValueError( + "Cannot determine time_origin for dummy UV fields. Assert T or o2 exists in fieldset." + ) from None From 813a245a159547db42cb8bce58f7e36e1ac55076 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 3 Nov 2025 11:22:33 +0100 Subject: [PATCH 38/97] Neaten up logging output --- src/virtualship/expedition/do_expedition.py | 6 +++++ src/virtualship/instruments/adcp.py | 3 ++- src/virtualship/instruments/argo_float.py | 4 +++- src/virtualship/instruments/base.py | 22 ++++++++++++++----- src/virtualship/instruments/ctd.py | 3 ++- src/virtualship/instruments/ctd_bgc.py | 4 +++- src/virtualship/instruments/drifter.py | 3 ++- .../instruments/ship_underwater_st.py | 3 ++- src/virtualship/instruments/xbt.py | 3 ++- 9 files changed, 38 insertions(+), 13 deletions(-) diff --git a/src/virtualship/expedition/do_expedition.py b/src/virtualship/expedition/do_expedition.py index ec61b1b08..b857b72c7 100644 --- a/src/virtualship/expedition/do_expedition.py +++ b/src/virtualship/expedition/do_expedition.py @@ -1,5 +1,6 @@ """do_expedition function.""" +import logging import os import shutil from pathlib import Path @@ -26,6 +27,11 @@ projection = pyproj.Geod(ellps="WGS84") +# parcels logger (suppress INFO messages to prevent log being flooded) +external_logger = logging.getLogger("parcels.tools.loggers") +external_logger.setLevel(logging.WARNING) + + def do_expedition(expedition_dir: str | Path) -> None: """ Perform an expedition, providing terminal feedback and file output. diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index db5be8eff..20a124361 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -89,6 +89,7 @@ def __init__(self, expedition, directory): variables, add_bathymetry=False, allow_time_extrapolation=True, + verbose_progress=False, ) def simulate(self, measurements, out_path) -> None: @@ -127,6 +128,6 @@ def simulate(self, measurements, out_path) -> None: [_sample_velocity], dt=1, runtime=1, - verbose_progress=False, + verbose_progress=self.verbose_progress, output_file=out_file, ) diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index 93aa90936..898fd5cc3 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -138,6 +138,7 @@ def __init__(self, data_dir, credentials, space_time_region): data_dir, credentials, space_time_region, + verbose_progress=True, ) def get_datasets_dict(self) -> dict: @@ -182,6 +183,7 @@ def __init__(self, expedition, directory): variables, add_bathymetry=False, allow_time_extrapolation=False, + verbose_progress=True, ) def simulate(self, measurements, out_path) -> None: @@ -243,5 +245,5 @@ def simulate(self, measurements, out_path) -> None: endtime=actual_endtime, dt=DT, output_file=out_file, - verbose_progress=True, + verbose_progress=self.verbose_progress, ) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 6c036b288..8212aab74 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -224,6 +224,7 @@ def __init__( variables: dict, add_bathymetry: bool, allow_time_extrapolation: bool, + verbose_progress: bool, bathymetry_file: str = "bathymetry.nc", ): """Initialise instrument.""" @@ -241,11 +242,16 @@ def __init__( self.bathymetry_file = bathymetry_file self.add_bathymetry = add_bathymetry self.allow_time_extrapolation = allow_time_extrapolation + self.verbose_progress = verbose_progress def load_input_data(self) -> FieldSet: """Load and return the input data as a FieldSet for the instrument.""" # TODO: can simulate_schedule.py be refactored to be contained in base.py and repsective instrument files too...? # TODO: tests need updating...! + + #! TODO: E.g. ADCP is giving too much depth data?! + #! TODO: in fact output from most instruments doesn't look quite right...? + try: data_dir = self._get_data_dir(self.directory) joined_filepaths = { @@ -290,13 +296,17 @@ def run(self, measurements: list, out_path: str | Path) -> None: """Run instrument simulation.""" # TODO: this will have to be able to handle the non-spinner/instead progress bar for drifters and argos! - with yaspin( - text=f"Simulating {self.name} measurements... ", - side="right", - spinner=ship_spinner, - ) as spinner: + if not self.verbose_progress: + with yaspin( + text=f"Simulating {self.name} measurements... ", + side="right", + spinner=ship_spinner, + ) as spinner: + self.simulate(measurements, out_path) + spinner.ok("✅") + else: + print(f"Simulating {self.name} measurements... ") self.simulate(measurements, out_path) - spinner.ok("✅") def _get_data_dir(self, expedition_dir: Path) -> Path: space_time_region_hash = get_space_time_region_hash( diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 22c69799f..9b59a7706 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -121,6 +121,7 @@ def __init__(self, expedition, directory): variables, add_bathymetry=True, allow_time_extrapolation=True, + verbose_progress=False, ) def simulate(self, measurements, out_path) -> None: @@ -199,7 +200,7 @@ def simulate(self, measurements, out_path) -> None: [_sample_salinity, _sample_temperature, _ctd_cast], endtime=fieldset_endtime, dt=DT, - verbose_progress=False, + verbose_progress=self.verbose_progress, output_file=out_file, ) diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index c42458234..397f1af43 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -100,6 +100,7 @@ def __init__(self, data_dir, credentials, space_time_region): data_dir, credentials, space_time_region, + verbose_progress=False, ) def get_datasets_dict(self) -> dict: @@ -175,6 +176,7 @@ def __init__(self, expedition, directory): variables, add_bathymetry=True, allow_time_extrapolation=True, + verbose_progress=False, ) def simulate(self, measurements, out_path) -> None: @@ -262,7 +264,7 @@ def simulate(self, measurements, out_path) -> None: ], endtime=fieldset_endtime, dt=DT, - verbose_progress=False, + verbose_progress=self.verbose_progress, output_file=out_file, ) diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index f33fde553..3b047fed5 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -102,6 +102,7 @@ def __init__(self, expedition, directory): variables, add_bathymetry=False, allow_time_extrapolation=False, + verbose_progress=True, ) def simulate(self, measurements, out_path) -> None: @@ -159,7 +160,7 @@ def simulate(self, measurements, out_path) -> None: endtime=actual_endtime, dt=DT, output_file=out_file, - verbose_progress=True, + verbose_progress=self.verbose_progress, ) # if there are more particles left than the number of drifters with an indefinite endtime, warn the user diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index f23937784..b52887a23 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -94,6 +94,7 @@ def __init__(self, expedition, directory): variables, add_bathymetry=False, allow_time_extrapolation=True, + verbose_progress=False, ) def simulate(self, measurements, out_path) -> None: @@ -129,6 +130,6 @@ def simulate(self, measurements, out_path) -> None: [_sample_salinity, _sample_temperature], dt=1, runtime=1, - verbose_progress=False, + verbose_progress=self.verbose_progress, output_file=out_file, ) diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index b15c0e6fd..641015f52 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -122,6 +122,7 @@ def __init__(self, expedition, directory): variables, add_bathymetry=True, allow_time_extrapolation=True, + verbose_progress=False, ) def simulate(self, measurements, out_path) -> None: @@ -193,7 +194,7 @@ def simulate(self, measurements, out_path) -> None: [_sample_temperature, _xbt_cast], endtime=fieldset_endtime, dt=DT, - verbose_progress=False, + verbose_progress=self.verbose_progress, output_file=out_file, ) From b94f4f0b1b7c43caf4d72e5f9762494b1366c42b Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 3 Nov 2025 13:14:26 +0100 Subject: [PATCH 39/97] small bug fixes --- src/virtualship/instruments/argo_float.py | 1 - src/virtualship/instruments/ctd_bgc.py | 1 - 2 files changed, 2 deletions(-) diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index 898fd5cc3..4a5dcdef2 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -138,7 +138,6 @@ def __init__(self, data_dir, credentials, space_time_region): data_dir, credentials, space_time_region, - verbose_progress=True, ) def get_datasets_dict(self) -> dict: diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 397f1af43..537ba810f 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -100,7 +100,6 @@ def __init__(self, data_dir, credentials, space_time_region): data_dir, credentials, space_time_region, - verbose_progress=False, ) def get_datasets_dict(self) -> dict: From a5d10e7d4ab9e417c195c460d41c7ba9944b1718 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 3 Nov 2025 13:43:00 +0100 Subject: [PATCH 40/97] tidy up --- src/virtualship/instruments/adcp.py | 25 +++++++++++++++++-- src/virtualship/instruments/argo_float.py | 22 ++++++++++++++++ src/virtualship/instruments/base.py | 7 ++---- src/virtualship/instruments/ctd.py | 24 +++++++++++++++--- src/virtualship/instruments/ctd_bgc.py | 22 ++++++++++++++++ src/virtualship/instruments/drifter.py | 22 ++++++++++++++++ .../instruments/ship_underwater_st.py | 22 ++++++++++++++++ src/virtualship/instruments/xbt.py | 22 ++++++++++++++++ src/virtualship/models/expedition.py | 1 - 9 files changed, 156 insertions(+), 11 deletions(-) diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index 20a124361..b2ebe988d 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -11,6 +11,10 @@ register_instrument, ) +# ===================================================== +# SECTION: Dataclass +# ===================================================== + @dataclass class ADCP: @@ -19,8 +23,11 @@ class ADCP: name: ClassVar[str] = "ADCP" -# we specifically use ScipyParticle because we have many small calls to execute -# there is some overhead with JITParticle and this ends up being significantly faster +# ===================================================== +# SECTION: Particle Class +# ===================================================== + + _ADCPParticle = ScipyParticle.add_variables( [ Variable("U", dtype=np.float32, initial=np.nan), @@ -28,6 +35,10 @@ class ADCP: ] ) +# ===================================================== +# SECTION: Kernels +# ===================================================== + def _sample_velocity(particle, fieldset, time): particle.U, particle.V = fieldset.UV.eval( @@ -35,6 +46,11 @@ def _sample_velocity(particle, fieldset, time): ) +# ===================================================== +# SECTION: InputDataset Class +# ===================================================== + + @register_input_dataset(InstrumentType.ADCP) class ADCPInputDataset(InputDataset): """Input dataset for ADCP instrument.""" @@ -70,6 +86,11 @@ def get_datasets_dict(self) -> dict: } +# ===================================================== +# SECTION: Instrument Class +# ===================================================== + + @register_instrument(InstrumentType.ADCP) class ADCPInstrument(Instrument): """ADCP instrument class.""" diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index 4a5dcdef2..573836b45 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -17,6 +17,10 @@ from virtualship.models.spacetime import Spacetime from virtualship.utils import register_input_dataset, register_instrument +# ===================================================== +# SECTION: Dataclass +# ===================================================== + @dataclass class ArgoFloat: @@ -32,6 +36,10 @@ class ArgoFloat: drift_days: float +# ===================================================== +# SECTION: Particle Class +# ===================================================== + _ArgoParticle = JITParticle.add_variables( [ Variable("cycle_phase", dtype=np.int32, initial=0.0), @@ -48,6 +56,10 @@ class ArgoFloat: ] ) +# ===================================================== +# SECTION: Kernels +# ===================================================== + def _argo_float_vertical_movement(particle, fieldset, time): if particle.cycle_phase == 0: @@ -116,6 +128,11 @@ def _check_error(particle, fieldset, time): particle.delete() +# ===================================================== +# SECTION: InputDataset Class +# ===================================================== + + @register_input_dataset(InstrumentType.ARGO_FLOAT) class ArgoFloatInputDataset(InputDataset): """Input dataset for ArgoFloat instrument.""" @@ -161,6 +178,11 @@ def get_datasets_dict(self) -> dict: } +# ===================================================== +# SECTION: Instrument Class +# ===================================================== + + @register_instrument(InstrumentType.ARGO_FLOAT) class ArgoFloatInstrument(Instrument): """ArgoFloat instrument class.""" diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 8212aab74..3c6a333b0 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -246,12 +246,8 @@ def __init__( def load_input_data(self) -> FieldSet: """Load and return the input data as a FieldSet for the instrument.""" - # TODO: can simulate_schedule.py be refactored to be contained in base.py and repsective instrument files too...? # TODO: tests need updating...! - #! TODO: E.g. ADCP is giving too much depth data?! - #! TODO: in fact output from most instruments doesn't look quite right...? - try: data_dir = self._get_data_dir(self.directory) joined_filepaths = { @@ -303,10 +299,11 @@ def run(self, measurements: list, out_path: str | Path) -> None: spinner=ship_spinner, ) as spinner: self.simulate(measurements, out_path) - spinner.ok("✅") + spinner.ok("✅\n") else: print(f"Simulating {self.name} measurements... ") self.simulate(measurements, out_path) + print("\n") def _get_data_dir(self, expedition_dir: Path) -> Path: space_time_region_hash = get_space_time_region_hash( diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 9b59a7706..5080c0060 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -13,6 +13,10 @@ # TODO: add some kind of check that each instrument has a dataclass, particle class, InputDataset class and Instrument class? # TODO: probably as a test +# ===================================================== +# SECTION: Dataclass +# ===================================================== + @dataclass class CTD: @@ -24,6 +28,10 @@ class CTD: max_depth: float +# ===================================================== +# SECTION: Particle Class +# ===================================================== + _CTDParticle = JITParticle.add_variables( [ Variable("salinity", dtype=np.float32, initial=np.nan), @@ -36,7 +44,9 @@ class CTD: ) -# TODO: way to group kernels together, just to make clearer? +# ===================================================== +# SECTION: Kernels +# ===================================================== def _sample_temperature(particle, fieldset, time): @@ -61,6 +71,11 @@ def _ctd_cast(particle, fieldset, time): particle.delete() +# ===================================================== +# SECTION: InputDataset Class +# ===================================================== + + @register_input_dataset(InstrumentType.CTD) class CTDInputDataset(InputDataset): """Input dataset for CTD instrument.""" @@ -99,14 +114,17 @@ def get_datasets_dict(self) -> dict: } +# ===================================================== +# SECTION: Instrument Class +# ===================================================== + + @register_instrument(InstrumentType.CTD) class CTDInstrument(Instrument): """CTD instrument class.""" def __init__(self, expedition, directory): """Initialize CTDInstrument.""" - #! TODO: actually don't need to download U and V for CTD simulation... can instead add mock/duplicate of T and name it U (also don't need V)! - filenames = { "S": f"{CTD.name}_s.nc", "T": f"{CTD.name}_t.nc", diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 537ba810f..28dd55c3f 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -10,6 +10,10 @@ from virtualship.models.spacetime import Spacetime from virtualship.utils import add_dummy_UV, register_input_dataset, register_instrument +# ===================================================== +# SECTION: Dataclass +# ===================================================== + @dataclass class CTD_BGC: @@ -21,6 +25,10 @@ class CTD_BGC: max_depth: float +# ===================================================== +# SECTION: Particle Class +# ===================================================== + _CTD_BGCParticle = JITParticle.add_variables( [ Variable("o2", dtype=np.float32, initial=np.nan), @@ -37,6 +45,10 @@ class CTD_BGC: ] ) +# ===================================================== +# SECTION: Kernels +# ===================================================== + def _sample_o2(particle, fieldset, time): particle.o2 = fieldset.o2[time, particle.depth, particle.lat, particle.lon] @@ -80,6 +92,11 @@ def _ctd_bgc_cast(particle, fieldset, time): particle.delete() +# ===================================================== +# SECTION: InputDataset Class +# ===================================================== + + @register_input_dataset(InstrumentType.CTD_BGC) class CTD_BGCInputDataset(InputDataset): """Input dataset object for CTD_BGC instrument.""" @@ -143,6 +160,11 @@ def get_datasets_dict(self) -> dict: } +# ===================================================== +# SECTION: Instrument Class +# ===================================================== + + @register_instrument(InstrumentType.CTD_BGC) class CTD_BGCInstrument(Instrument): """CTD_BGC instrument class.""" diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index 3b047fed5..0ce786243 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -10,6 +10,10 @@ from virtualship.models.spacetime import Spacetime from virtualship.utils import register_input_dataset, register_instrument +# ===================================================== +# SECTION: Dataclass +# ===================================================== + @dataclass class Drifter: @@ -21,6 +25,10 @@ class Drifter: lifetime: timedelta | None # if none, lifetime is infinite +# ===================================================== +# SECTION: Particle Class +# ===================================================== + _DrifterParticle = JITParticle.add_variables( [ Variable("temperature", dtype=np.float32, initial=np.nan), @@ -30,6 +38,10 @@ class Drifter: ] ) +# ===================================================== +# SECTION: Kernels +# ===================================================== + def _sample_temperature(particle, fieldset, time): particle.temperature = fieldset.T[time, particle.depth, particle.lat, particle.lon] @@ -42,6 +54,11 @@ def _check_lifetime(particle, fieldset, time): particle.delete() +# ===================================================== +# SECTION: InputDataset Class +# ===================================================== + + @register_input_dataset(InstrumentType.DRIFTER) class DrifterInputDataset(InputDataset): """Input dataset for Drifter instrument.""" @@ -82,6 +99,11 @@ def get_datasets_dict(self) -> dict: } +# ===================================================== +# SECTION: Instrument Class +# ===================================================== + + @register_instrument(InstrumentType.DRIFTER) class DrifterInstrument(Instrument): """Drifter instrument class.""" diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index b52887a23..946e1a9bc 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -8,6 +8,10 @@ from virtualship.instruments.types import InstrumentType from virtualship.utils import add_dummy_UV, register_input_dataset, register_instrument +# ===================================================== +# SECTION: Dataclass +# ===================================================== + @dataclass class Underwater_ST: @@ -16,6 +20,10 @@ class Underwater_ST: name: ClassVar[str] = "Underwater_ST" +# ===================================================== +# SECTION: Particle Class +# ===================================================== + _ShipSTParticle = ScipyParticle.add_variables( [ Variable("S", dtype=np.float32, initial=np.nan), @@ -23,6 +31,10 @@ class Underwater_ST: ] ) +# ===================================================== +# SECTION: Kernels +# ===================================================== + # define function sampling Salinity def _sample_salinity(particle, fieldset, time): @@ -34,6 +46,11 @@ def _sample_temperature(particle, fieldset, time): particle.T = fieldset.T[time, particle.depth, particle.lat, particle.lon] +# ===================================================== +# SECTION: InputDataset Class +# ===================================================== + + @register_input_dataset(InstrumentType.UNDERWATER_ST) class Underwater_STInputDataset(InputDataset): """Input dataset for Underwater_ST instrument.""" @@ -74,6 +91,11 @@ def get_datasets_dict(self) -> dict: } +# ===================================================== +# SECTION: Instrument Class +# ===================================================== + + @register_instrument(InstrumentType.UNDERWATER_ST) class Underwater_STInstrument(Instrument): """Underwater_ST instrument class.""" diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index 641015f52..98378c988 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -10,6 +10,10 @@ from virtualship.models.spacetime import Spacetime from virtualship.utils import register_input_dataset, register_instrument +# ===================================================== +# SECTION: Dataclass +# ===================================================== + @dataclass class XBT: @@ -23,6 +27,10 @@ class XBT: deceleration_coefficient: float +# ===================================================== +# SECTION: Particle Class +# ===================================================== + _XBTParticle = JITParticle.add_variables( [ Variable("temperature", dtype=np.float32, initial=np.nan), @@ -33,6 +41,10 @@ class XBT: ] ) +# ===================================================== +# SECTION: Kernels +# ===================================================== + def _sample_temperature(particle, fieldset, time): particle.temperature = fieldset.T[time, particle.depth, particle.lat, particle.lon] @@ -56,6 +68,11 @@ def _xbt_cast(particle, fieldset, time): particle_ddepth = particle.max_depth - particle.depth +# ===================================================== +# SECTION: InputDataset Class +# ===================================================== + + @register_input_dataset(InstrumentType.XBT) class XBTInputDataset(InputDataset): """Input dataset for XBT instrument.""" @@ -101,6 +118,11 @@ def get_datasets_dict(self) -> dict: } +# ===================================================== +# SECTION: Instrument Class +# ===================================================== + + @register_instrument(InstrumentType.XBT) class XBTInstrument(Instrument): """XBT instrument class.""" diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index c70de855b..540211769 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -341,7 +341,6 @@ class XBTConfig(pydantic.BaseModel): class InstrumentsConfig(pydantic.BaseModel): - # TODO: refactor potential for this? Move explicit instrument_config's away from models/ dir? """Configuration of instruments.""" argo_float_config: ArgoFloatConfig | None = None From 37530329840f87ad08e1e7c0bd067a7fdcbc9515 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 3 Nov 2025 12:50:10 +0000 Subject: [PATCH 41/97] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/virtualship/instruments/adcp.py | 2 +- src/virtualship/instruments/argo_float.py | 2 +- src/virtualship/instruments/base.py | 2 +- src/virtualship/instruments/ctd.py | 2 +- src/virtualship/instruments/ctd_bgc.py | 2 +- src/virtualship/instruments/drifter.py | 2 +- src/virtualship/instruments/ship_underwater_st.py | 2 +- src/virtualship/instruments/xbt.py | 2 +- src/virtualship/models/expedition.py | 2 +- tests/instruments/test_ctd.py | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index b2ebe988d..857e96555 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -2,8 +2,8 @@ from typing import ClassVar import numpy as np - from parcels import ParticleSet, ScipyParticle, Variable + from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.utils import ( diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index 573836b45..649616052 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -4,7 +4,6 @@ from typing import ClassVar import numpy as np - from parcels import ( AdvectionRK4, JITParticle, @@ -12,6 +11,7 @@ StatusCode, Variable, ) + from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 3c6a333b0..93ff8ed8b 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -4,9 +4,9 @@ import copernicusmarine import numpy as np +from parcels import Field, FieldSet from yaspin import yaspin -from parcels import Field, FieldSet from virtualship.cli._fetch import get_existing_download, get_space_time_region_hash from virtualship.errors import CopernicusCatalogueError from virtualship.models import Expedition, SpaceTimeRegion diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 5080c0060..2c6dc56eb 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np - from parcels import JITParticle, ParticleSet, Variable + from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.models import Spacetime diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 28dd55c3f..23c978f70 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np - from parcels import JITParticle, ParticleSet, Variable + from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index 0ce786243..f854d51f9 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np - from parcels import AdvectionRK4, JITParticle, ParticleSet, Variable + from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index 946e1a9bc..161bb1843 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -2,8 +2,8 @@ from typing import ClassVar import numpy as np - from parcels import ParticleSet, ScipyParticle, Variable + from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.utils import add_dummy_UV, register_input_dataset, register_instrument diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index 98378c988..68502533c 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np - from parcels import JITParticle, ParticleSet, Variable + from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index 540211769..d6727f699 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -9,8 +9,8 @@ import pydantic import pyproj import yaml - from parcels import Field + from virtualship.errors import InstrumentsConfigError, ScheduleError from virtualship.instruments.types import InstrumentType from virtualship.utils import _validate_numeric_mins_to_timedelta diff --git a/tests/instruments/test_ctd.py b/tests/instruments/test_ctd.py index 0a8edcfa0..14e0a2765 100644 --- a/tests/instruments/test_ctd.py +++ b/tests/instruments/test_ctd.py @@ -9,8 +9,8 @@ import numpy as np import xarray as xr - from parcels import Field, FieldSet + from virtualship.instruments.ctd import CTD, simulate_ctd from virtualship.models import Location, Spacetime From b7904e8995cdeacf53538623e5efdf51d7268f8e Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 3 Nov 2025 14:38:10 +0100 Subject: [PATCH 42/97] Refactor type hints and improve test coverage for instruments and utils --- src/virtualship/instruments/base.py | 41 +++++++------ src/virtualship/instruments/ctd.py | 8 ++- tests/cli/test_fetch.py | 27 --------- tests/instruments/test_base.py | 92 +++++++++++++++++++++-------- tests/test_utils.py | 18 +++++- 5 files changed, 113 insertions(+), 73 deletions(-) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 3c6a333b0..0bb77c334 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -1,6 +1,7 @@ import abc from datetime import timedelta from pathlib import Path +from typing import TYPE_CHECKING import copernicusmarine import numpy as np @@ -9,9 +10,11 @@ from parcels import Field, FieldSet from virtualship.cli._fetch import get_existing_download, get_space_time_region_hash from virtualship.errors import CopernicusCatalogueError -from virtualship.models import Expedition, SpaceTimeRegion from virtualship.utils import ship_spinner +if TYPE_CHECKING: + from virtualship.models import Expedition, SpaceTimeRegion + PRODUCT_IDS = { "phys": { "reanalysis": "cmems_mod_glo_phy_my_0.083deg_P1D-m", @@ -60,7 +63,7 @@ def __init__( max_depth: float, data_dir: str, credentials: dict, - space_time_region: SpaceTimeRegion, + space_time_region: "SpaceTimeRegion", ): """Initialise input dataset.""" self.name = name @@ -185,22 +188,7 @@ def _select_product_id( "No suitable product found in the Copernicus Marine Catalogue for the scheduled time and variable." ) - def start_end_in_product_timerange( - selected_id, schedule_start, schedule_end, username, password - ): - ds_selected = copernicusmarine.open_dataset( - selected_id, username=username, password=password - ) - time_values = ds_selected["time"].values - import numpy as np - - time_min, time_max = np.min(time_values), np.max(time_values) - return ( - np.datetime64(schedule_start) >= time_min - and np.datetime64(schedule_end) <= time_max - ) - - if start_end_in_product_timerange( + if self._start_end_in_product_timerange( selected_id, schedule_start, schedule_end, username, password ): return selected_id @@ -211,6 +199,21 @@ def start_end_in_product_timerange( else BGC_ANALYSIS_IDS[variable] ) + def _start_end_in_product_timerange( + self, selected_id, schedule_start, schedule_end, username, password + ): + ds_selected = copernicusmarine.open_dataset( + selected_id, username=username, password=password + ) + time_values = ds_selected["time"].values + import numpy as np + + time_min, time_max = np.min(time_values), np.max(time_values) + return ( + np.datetime64(schedule_start) >= time_min + and np.datetime64(schedule_end) <= time_max + ) + class Instrument(abc.ABC): """Base class for instruments and their simulation.""" @@ -218,7 +221,7 @@ class Instrument(abc.ABC): def __init__( self, name: str, - expedition: Expedition, + expedition: "Expedition", directory: Path | str, filenames: dict, variables: dict, diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 5080c0060..8e524f4aa 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -1,13 +1,15 @@ from dataclasses import dataclass from datetime import timedelta -from typing import ClassVar +from typing import TYPE_CHECKING, ClassVar import numpy as np from parcels import JITParticle, ParticleSet, Variable from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType -from virtualship.models import Spacetime + +if TYPE_CHECKING: + from virtualship.models.spacetime import Spacetime from virtualship.utils import add_dummy_UV, register_input_dataset, register_instrument # TODO: add some kind of check that each instrument has a dataclass, particle class, InputDataset class and Instrument class? @@ -23,7 +25,7 @@ class CTD: """CTD configuration.""" name: ClassVar[str] = "CTD" - spacetime: Spacetime + spacetime: "Spacetime" min_depth: float max_depth: float diff --git a/tests/cli/test_fetch.py b/tests/cli/test_fetch.py index 9725601a3..4c041dbe5 100644 --- a/tests/cli/test_fetch.py +++ b/tests/cli/test_fetch.py @@ -17,8 +17,6 @@ get_existing_download, hash_model, hash_to_filename, - select_product_id, - start_end_in_product_timerange, ) from virtualship.models import Expedition from virtualship.utils import EXPEDITION, get_example_expedition @@ -98,31 +96,6 @@ def test_complete_download(tmp_path): assert_complete_download(tmp_path) -@pytest.mark.usefixtures("copernicus_no_download") -def test_select_product_id(expedition): - """Should return the physical reanalysis product id via the timings prescribed in the static schedule.yaml file.""" - result = select_product_id( - physical=True, - schedule_start=expedition.schedule.space_time_region.time_range.start_time, - schedule_end=expedition.schedule.space_time_region.time_range.end_time, - username="test", - password="test", - ) - assert result == "cmems_mod_glo_phy_my_0.083deg_P1D-m" - - -@pytest.mark.usefixtures("copernicus_no_download") -def test_start_end_in_product_timerange(expedition): - """Should return True for valid range ass determined by the static schedule.yaml file.""" - assert start_end_in_product_timerange( - selected_id="cmems_mod_glo_phy_my_0.083deg_P1D-m", - schedule_start=expedition.schedule.space_time_region.time_range.start_time, - schedule_end=expedition.schedule.space_time_region.time_range.end_time, - username="test", - password="test", - ) - - def test_assert_complete_download_complete(tmp_path): # Setup DownloadMetadata(download_complete=True).to_yaml(tmp_path / DOWNLOAD_METADATA) diff --git a/tests/instruments/test_base.py b/tests/instruments/test_base.py index f41920924..65e4ae0ed 100644 --- a/tests/instruments/test_base.py +++ b/tests/instruments/test_base.py @@ -1,9 +1,12 @@ import datetime -from unittest.mock import patch +from pathlib import Path +from unittest.mock import MagicMock, patch +import numpy as np import pytest +import xarray as xr -from virtualship.instruments.base import InputDataset +from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.space_time_region import ( SpaceTimeRegion, @@ -12,6 +15,18 @@ ) from virtualship.utils import get_input_dataset_class +# test dataclass, particle class, kernels, etc. are defined for each instrument + + +# TODO: add all the other things here like particle class, kernels, etc. +def test_all_instruments_have_input_class(): + for instrument in InstrumentType: + input_class = get_input_dataset_class(instrument) + assert input_class is not None, f"No input_class for {instrument}" + + +# test InputDataset class + class DummyInputDataset(InputDataset): """A minimal InputDataset subclass for testing purposes.""" @@ -20,7 +35,7 @@ def get_datasets_dict(self): """Return a dummy datasets dict for testing.""" return { "dummy": { - "dataset_id": "test_id", + "physical": True, "variables": ["var1"], "output_filename": "dummy.nc", } @@ -48,21 +63,6 @@ def dummy_space_time_region(): ) -def test_inputdataset_abstract_instantiation(): - # instantiation should not be allowed - with pytest.raises(TypeError): - InputDataset( - name="test", - latlon_buffer=0, - datetime_buffer=0, - min_depth=0, - max_depth=10, - data_dir=".", - credentials={"username": "u", "password": "p"}, - space_time_region=None, - ) - - def test_dummyinputdataset_initialization(dummy_space_time_region): ds = DummyInputDataset( name="test", @@ -83,8 +83,23 @@ def test_dummyinputdataset_initialization(dummy_space_time_region): assert ds.credentials["username"] == "u" +@patch("virtualship.instruments.base.copernicusmarine.open_dataset") @patch("virtualship.instruments.base.copernicusmarine.subset") -def test_download_data_calls_subset(mock_subset, dummy_space_time_region): +def test_download_data_calls_subset( + mock_subset, mock_open_dataset, dummy_space_time_region +): + """Test that download_data calls the subset function correctly, will also test Copernicus Marine product id search logic.""" + mock_open_dataset.return_value = xr.Dataset( + { + "time": ( + "time", + [ + np.datetime64("1993-01-01T00:00:00"), + np.datetime64("2023-01-01T01:00:00"), + ], + ) + } + ) ds = DummyInputDataset( name="test", latlon_buffer=0.5, @@ -99,7 +114,38 @@ def test_download_data_calls_subset(mock_subset, dummy_space_time_region): assert mock_subset.called -def test_all_instruments_have_input_class(): - for instrument in InstrumentType: - input_class = get_input_dataset_class(instrument) - assert input_class is not None, f"No input_class for {instrument}" +# test Instrument class + + +class DummyInstrument(Instrument): + """Minimal concrete Instrument for testing.""" + + def simulate(self, data_dir, measurements, out_path): + """Dummy simulate implementation for test.""" + self.simulate_called = True + + +@patch("virtualship.instruments.base.FieldSet") +@patch("virtualship.instruments.base.get_existing_download") +@patch("virtualship.instruments.base.get_space_time_region_hash") +def test_load_input_data_calls(mock_hash, mock_get_download, mock_FieldSet): + """Test Instrument.load_input_data with mocks.""" + mock_hash.return_value = "hash" + mock_get_download.return_value = Path("/tmp/data") + mock_fieldset = MagicMock() + mock_FieldSet.from_netcdf.return_value = mock_fieldset + mock_fieldset.gridset.grids = [MagicMock(negate_depth=MagicMock())] + mock_fieldset.__getitem__.side_effect = lambda k: MagicMock() + dummy = DummyInstrument( + name="test", + expedition=MagicMock(schedule=MagicMock(space_time_region=MagicMock())), + directory="/tmp", + filenames={"A": "a.nc"}, + variables={"A": "a"}, + add_bathymetry=False, + allow_time_extrapolation=False, + verbose_progress=False, + ) + fieldset = dummy.load_input_data() + assert mock_FieldSet.from_netcdf.called + assert fieldset == mock_fieldset diff --git a/tests/test_utils.py b/tests/test_utils.py index 0dcebd794..bb8208f65 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,4 +1,4 @@ -from virtualship.models import Expedition +from virtualship.models.expedition import Expedition from virtualship.utils import get_example_expedition @@ -12,3 +12,19 @@ def test_valid_example_expedition(tmp_path): file.write(get_example_expedition()) Expedition.from_yaml(path) + + +def test_instrument_registry_updates(): + from virtualship import utils + + class DummyInputDataset: + pass + + class DummyInstrument: + pass + + utils.register_input_dataset("DUMMY_TYPE")(DummyInputDataset) + utils.register_instrument("DUMMY_TYPE")(DummyInstrument) + + assert utils.INPUT_DATASET_MAP["DUMMY_TYPE"] is DummyInputDataset + assert utils.INSTRUMENT_CLASS_MAP["DUMMY_TYPE"] is DummyInstrument From 7d1c575c7b8bc309db7e6e147357ab83a6c19f7a Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 3 Nov 2025 15:05:31 +0100 Subject: [PATCH 43/97] Remove TODO comments and tidy up imports in test files --- src/virtualship/instruments/ctd.py | 3 --- src/virtualship/models/expedition.py | 1 - tests/instruments/test_base.py | 1 - 3 files changed, 5 deletions(-) diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 8e524f4aa..0164ef945 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -12,9 +12,6 @@ from virtualship.models.spacetime import Spacetime from virtualship.utils import add_dummy_UV, register_input_dataset, register_instrument -# TODO: add some kind of check that each instrument has a dataclass, particle class, InputDataset class and Instrument class? -# TODO: probably as a test - # ===================================================== # SECTION: Dataclass # ===================================================== diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index 540211769..dc198f511 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -131,7 +131,6 @@ def verify( ) # check if all waypoints are in water using bathymetry data - # TODO: tests should be updated to check this! # TODO: write test that checks that will flag when waypoint is on land!! [add to existing suite of fail .verify() tests in test_expedition.py] # TODO: need to do an overhaul of the DATA which is in tests/expedition/expedition_dir - don't think it's currently suitable! land_waypoints = [] diff --git a/tests/instruments/test_base.py b/tests/instruments/test_base.py index 65e4ae0ed..0a58efcd9 100644 --- a/tests/instruments/test_base.py +++ b/tests/instruments/test_base.py @@ -18,7 +18,6 @@ # test dataclass, particle class, kernels, etc. are defined for each instrument -# TODO: add all the other things here like particle class, kernels, etc. def test_all_instruments_have_input_class(): for instrument in InstrumentType: input_class = get_input_dataset_class(instrument) From e74a5ada099226c60704b991af42397e1e5bd634 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 3 Nov 2025 15:26:15 +0100 Subject: [PATCH 44/97] Refactor bathymetry error handling, update verification methods and remove unused function --- src/virtualship/models/expedition.py | 40 ++++++++-------------------- 1 file changed, 11 insertions(+), 29 deletions(-) diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index 7c109c99e..af85c1c53 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -5,12 +5,11 @@ from pathlib import Path from typing import TYPE_CHECKING -import numpy as np import pydantic import pyproj import yaml -from parcels import Field +from parcels import Field from virtualship.errors import InstrumentsConfigError, ScheduleError from virtualship.instruments.types import InstrumentType from virtualship.utils import _validate_numeric_mins_to_timedelta @@ -19,7 +18,7 @@ from .space_time_region import SpaceTimeRegion if TYPE_CHECKING: - from parcels import FieldSet + pass projection: pyproj.Geod = pyproj.Geod(ellps="WGS84") @@ -132,7 +131,6 @@ def verify( # check if all waypoints are in water using bathymetry data # TODO: write test that checks that will flag when waypoint is on land!! [add to existing suite of fail .verify() tests in test_expedition.py] - # TODO: need to do an overhaul of the DATA which is in tests/expedition/expedition_dir - don't think it's currently suitable! land_waypoints = [] if data_dir is not None: bathymetry_path = data_dir.joinpath("bathymetry.nc") @@ -147,18 +145,19 @@ def verify( f"Problem loading bathymetry data (used to verify waypoints are in water): {e}" ) from e for wp_i, wp in enumerate(self.waypoints): - bathy = bathymetry_field.eval( - 0, # time - 0, # depth (surface) - wp.location.lat, - wp.location.lon, - ) - if np.isnan(bathy) or bathy <= 0: + try: + bathymetry_field.eval( + 0, # time + 0, # depth (surface) + wp.location.lat, + wp.location.lon, + ) + except Exception: land_waypoints.append((wp_i, wp)) if len(land_waypoints) > 0: raise ScheduleError( - f"The following waypoints are on land: {['#' + str(wp_i + 1) + ' ' + str(wp) for (wp_i, wp) in land_waypoints]}" + f"The following waypoint(s) throw(s) error(s): {['#' + str(wp_i + 1) + ' ' + str(wp) for (wp_i, wp) in land_waypoints]}\n\nINFO: They are likely on land (bathymetry data cannot be interpolated to their location(s)).\n" ) elif not ignore_missing_bathymetry: raise ScheduleError( @@ -431,20 +430,3 @@ def verify(self, expedition: Expedition) -> None: raise InstrumentsConfigError( f"Expedition includes instrument '{inst_type.value}', but instruments_config does not provide configuration for it." ) - - -def _is_on_land_zero_uv(fieldset: FieldSet, waypoint: Waypoint) -> bool: - """ - Check if waypoint is on land by assuming zero velocity means land. - - :param fieldset: The fieldset to sample the velocity from. - :param waypoint: The waypoint to check. - :returns: If the waypoint is on land. - """ - return fieldset.UV.eval( - 0, - fieldset.gridset.grids[0].depth[0], - waypoint.location.lat, - waypoint.location.lon, - applyConversion=False, - ) == (0.0, 0.0) From 56d8fd580900e08126ebeb6a9106e5662b09c8d3 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 5 Nov 2025 12:53:49 +0100 Subject: [PATCH 45/97] move product id selection logic to utils --- src/virtualship/instruments/base.py | 128 +--------------------------- src/virtualship/utils.py | 124 +++++++++++++++++++++++++++ 2 files changed, 127 insertions(+), 125 deletions(-) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index a35aa81ab..0f5307029 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -4,49 +4,15 @@ from typing import TYPE_CHECKING import copernicusmarine -import numpy as np -from parcels import Field, FieldSet from yaspin import yaspin +from parcels import Field, FieldSet from virtualship.cli._fetch import get_existing_download, get_space_time_region_hash -from virtualship.errors import CopernicusCatalogueError -from virtualship.utils import ship_spinner +from virtualship.utils import _select_product_id, ship_spinner if TYPE_CHECKING: from virtualship.models import Expedition, SpaceTimeRegion -PRODUCT_IDS = { - "phys": { - "reanalysis": "cmems_mod_glo_phy_my_0.083deg_P1D-m", - "reanalysis_interim": "cmems_mod_glo_phy_myint_0.083deg_P1D-m", - "analysis": "cmems_mod_glo_phy_anfc_0.083deg_P1D-m", - }, - "bgc": { - "reanalysis": "cmems_mod_glo_bgc_my_0.25deg_P1D-m", - "reanalysis_interim": "cmems_mod_glo_bgc_myint_0.25deg_P1D-m", - "analysis": None, # will be set per variable - }, -} - -BGC_ANALYSIS_IDS = { - "o2": "cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m", - "chl": "cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m", - "no3": "cmems_mod_glo_bgc-nut_anfc_0.25deg_P1D-m", - "po4": "cmems_mod_glo_bgc-nut_anfc_0.25deg_P1D-m", - "ph": "cmems_mod_glo_bgc-car_anfc_0.25deg_P1D-m", - "phyc": "cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m", - "nppv": "cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m", -} - -MONTHLY_BGC_REANALYSIS_IDS = { - "ph": "cmems_mod_glo_bgc_my_0.25deg_P1M-m", - "phyc": "cmems_mod_glo_bgc_my_0.25deg_P1M-m", -} -MONTHLY_BGC_REANALYSIS_INTERIM_IDS = { - "ph": "cmems_mod_glo_bgc_myint_0.25deg_P1M-m", - "phyc": "cmems_mod_glo_bgc_myint_0.25deg_P1M-m", -} - class InputDataset(abc.ABC): """Base class for instrument input datasets.""" @@ -111,7 +77,7 @@ def download_data(self) -> None: else: variable = dataset.get("variables")[0] # BGC variables, special case - dataset_id = self._select_product_id( + dataset_id = _select_product_id( physical=physical, schedule_start=self.space_time_region.time_range.start_time, schedule_end=self.space_time_region.time_range.end_time, @@ -126,94 +92,6 @@ def download_data(self) -> None: } copernicusmarine.subset(**download_args) - def _select_product_id( - self, - physical: bool, - schedule_start, - schedule_end, - username: str, - password: str, - variable: str | None = None, - ) -> str: - """Determine which copernicus product id should be selected (reanalysis, reanalysis-interim, analysis & forecast), for prescribed schedule and physical vs. BGC.""" - key = "phys" if physical else "bgc" - selected_id = None - - for period, pid in PRODUCT_IDS[key].items(): - # for BGC analysis, set pid per variable - if key == "bgc" and period == "analysis": - if variable is None or variable not in BGC_ANALYSIS_IDS: - continue - pid = BGC_ANALYSIS_IDS[variable] - # for BGC reanalysis, check if requires monthly product - if ( - key == "bgc" - and period == "reanalysis" - and variable in MONTHLY_BGC_REANALYSIS_IDS - ): - monthly_pid = MONTHLY_BGC_REANALYSIS_IDS[variable] - ds_monthly = copernicusmarine.open_dataset( - monthly_pid, - username=username, - password=password, - ) - time_end_monthly = ds_monthly["time"][-1].values - if np.datetime64(schedule_end) <= time_end_monthly: - pid = monthly_pid - # for BGC reanalysis_interim, check if requires monthly product - if ( - key == "bgc" - and period == "reanalysis_interim" - and variable in MONTHLY_BGC_REANALYSIS_INTERIM_IDS - ): - monthly_pid = MONTHLY_BGC_REANALYSIS_INTERIM_IDS[variable] - ds_monthly = copernicusmarine.open_dataset( - monthly_pid, username=username, password=password - ) - time_end_monthly = ds_monthly["time"][-1].values - if np.datetime64(schedule_end) <= time_end_monthly: - pid = monthly_pid - if pid is None: - continue - ds = copernicusmarine.open_dataset( - pid, username=username, password=password - ) - time_end = ds["time"][-1].values - if np.datetime64(schedule_end) <= time_end: - selected_id = pid - break - - if selected_id is None: - raise CopernicusCatalogueError( - "No suitable product found in the Copernicus Marine Catalogue for the scheduled time and variable." - ) - - if self._start_end_in_product_timerange( - selected_id, schedule_start, schedule_end, username, password - ): - return selected_id - else: - return ( - PRODUCT_IDS["phys"]["analysis"] - if physical - else BGC_ANALYSIS_IDS[variable] - ) - - def _start_end_in_product_timerange( - self, selected_id, schedule_start, schedule_end, username, password - ): - ds_selected = copernicusmarine.open_dataset( - selected_id, username=username, password=password - ) - time_values = ds_selected["time"].values - import numpy as np - - time_min, time_max = np.min(time_values), np.max(time_values) - return ( - np.datetime64(schedule_start) >= time_min - and np.datetime64(schedule_end) <= time_max - ) - class Instrument(abc.ABC): """Base class for instruments and their simulation.""" diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index 324f8c672..8f1fadd47 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -8,7 +8,11 @@ from pathlib import Path from typing import TYPE_CHECKING, TextIO +import copernicusmarine +import numpy as np + from parcels import FieldSet +from virtualship.errors import CopernicusCatalogueError if TYPE_CHECKING: from virtualship.models import Expedition @@ -293,3 +297,123 @@ def add_dummy_UV(fieldset: FieldSet): raise ValueError( "Cannot determine time_origin for dummy UV fields. Assert T or o2 exists in fieldset." ) from None + + +# Copernicus Marine product IDs + +PRODUCT_IDS = { + "phys": { + "reanalysis": "cmems_mod_glo_phy_my_0.083deg_P1D-m", + "reanalysis_interim": "cmems_mod_glo_phy_myint_0.083deg_P1D-m", + "analysis": "cmems_mod_glo_phy_anfc_0.083deg_P1D-m", + }, + "bgc": { + "reanalysis": "cmems_mod_glo_bgc_my_0.25deg_P1D-m", + "reanalysis_interim": "cmems_mod_glo_bgc_myint_0.25deg_P1D-m", + "analysis": None, # will be set per variable + }, +} + +BGC_ANALYSIS_IDS = { + "o2": "cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m", + "chl": "cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m", + "no3": "cmems_mod_glo_bgc-nut_anfc_0.25deg_P1D-m", + "po4": "cmems_mod_glo_bgc-nut_anfc_0.25deg_P1D-m", + "ph": "cmems_mod_glo_bgc-car_anfc_0.25deg_P1D-m", + "phyc": "cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m", + "nppv": "cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m", +} + +MONTHLY_BGC_REANALYSIS_IDS = { + "ph": "cmems_mod_glo_bgc_my_0.25deg_P1M-m", + "phyc": "cmems_mod_glo_bgc_my_0.25deg_P1M-m", +} +MONTHLY_BGC_REANALYSIS_INTERIM_IDS = { + "ph": "cmems_mod_glo_bgc_myint_0.25deg_P1M-m", + "phyc": "cmems_mod_glo_bgc_myint_0.25deg_P1M-m", +} + + +def _select_product_id( + physical: bool, + schedule_start, + schedule_end, + username: str, + password: str, + variable: str | None = None, +) -> str: + """Determine which copernicus product id should be selected (reanalysis, reanalysis-interim, analysis & forecast), for prescribed schedule and physical vs. BGC.""" + key = "phys" if physical else "bgc" + selected_id = None + + for period, pid in PRODUCT_IDS[key].items(): + # for BGC analysis, set pid per variable + if key == "bgc" and period == "analysis": + if variable is None or variable not in BGC_ANALYSIS_IDS: + continue + pid = BGC_ANALYSIS_IDS[variable] + # for BGC reanalysis, check if requires monthly product + if ( + key == "bgc" + and period == "reanalysis" + and variable in MONTHLY_BGC_REANALYSIS_IDS + ): + monthly_pid = MONTHLY_BGC_REANALYSIS_IDS[variable] + ds_monthly = copernicusmarine.open_dataset( + monthly_pid, + username=username, + password=password, + ) + time_end_monthly = ds_monthly["time"][-1].values + if np.datetime64(schedule_end) <= time_end_monthly: + pid = monthly_pid + # for BGC reanalysis_interim, check if requires monthly product + if ( + key == "bgc" + and period == "reanalysis_interim" + and variable in MONTHLY_BGC_REANALYSIS_INTERIM_IDS + ): + monthly_pid = MONTHLY_BGC_REANALYSIS_INTERIM_IDS[variable] + ds_monthly = copernicusmarine.open_dataset( + monthly_pid, username=username, password=password + ) + time_end_monthly = ds_monthly["time"][-1].values + if np.datetime64(schedule_end) <= time_end_monthly: + pid = monthly_pid + if pid is None: + continue + ds = copernicusmarine.open_dataset(pid, username=username, password=password) + time_end = ds["time"][-1].values + if np.datetime64(schedule_end) <= time_end: + selected_id = pid + break + + if selected_id is None: + raise CopernicusCatalogueError( + "No suitable product found in the Copernicus Marine Catalogue for the scheduled time and variable." + ) + + if _start_end_in_product_timerange( + selected_id, schedule_start, schedule_end, username, password + ): + return selected_id + else: + return ( + PRODUCT_IDS["phys"]["analysis"] if physical else BGC_ANALYSIS_IDS[variable] + ) + + +def _start_end_in_product_timerange( + selected_id, schedule_start, schedule_end, username, password +): + ds_selected = copernicusmarine.open_dataset( + selected_id, username=username, password=password + ) + time_values = ds_selected["time"].values + import numpy as np + + time_min, time_max = np.min(time_values), np.max(time_values) + return ( + np.datetime64(schedule_start) >= time_min + and np.datetime64(schedule_end) <= time_max + ) From c447dd86c446e7a77e2aba2a87d6a6efdff9e47b Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 5 Nov 2025 14:11:46 +0100 Subject: [PATCH 46/97] update --- src/virtualship/instruments/base.py | 69 +++++++++++++++++++++-------- 1 file changed, 51 insertions(+), 18 deletions(-) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 0f5307029..0e5ff37a3 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -96,6 +96,11 @@ def download_data(self) -> None: class Instrument(abc.ABC): """Base class for instruments and their simulation.""" + #! TODO List: + # TODO: update documentation/quickstart + # TODO: update tests + # TODO: if use direct ingestion as primary data sourcing, can substantially cut code base (including _fetch.py, InputDataset objects). Consider this for Parcels v4 transition. + def __init__( self, name: str, @@ -107,6 +112,7 @@ def __init__( allow_time_extrapolation: bool, verbose_progress: bool, bathymetry_file: str = "bathymetry.nc", + direct: bool = False, ): """Initialise instrument.""" self.name = name @@ -124,27 +130,54 @@ def __init__( self.add_bathymetry = add_bathymetry self.allow_time_extrapolation = allow_time_extrapolation self.verbose_progress = verbose_progress + self.direct = direct def load_input_data(self) -> FieldSet: """Load and return the input data as a FieldSet for the instrument.""" - # TODO: tests need updating...! - - try: - data_dir = self._get_data_dir(self.directory) - joined_filepaths = { - key: data_dir.joinpath(filename) - for key, filename in self.filenames.items() - } - fieldset = FieldSet.from_netcdf( - joined_filepaths, - self.variables, - self.dimensions, - allow_time_extrapolation=self.allow_time_extrapolation, - ) - except FileNotFoundError as e: - raise FileNotFoundError( - f"Input data for instrument {self.name} not found. Have you run the `virtualship fetch` command??" - ) from e + if self.direct: # if direct ingestion from Copernicus Marine is enabled + try: + # ds = copernicusmarine.open_dataset( + # dataset_id="PHYS_REANALYSIS_ID", + # dataset_part="default", + # minimum_longitude=self.expedition.schedule.space_time_region.spatial_range.minimum_longitude, + # maximum_longitude=self.expedition.schedule.space_time_region.spatial_range.maximum_longitude, + # minimum_latitude=self.expedition.schedule.space_time_region.spatial_range.minimum_latitude, + # maximum_latitude=self.expedition.schedule.space_time_region.spatial_range.maximum_latitude, + # variables=["uo", "vo", "so", "thetao"], + # start_datetime=self.expedition.schedule.space_time_region.time_range.start_time, + # end_datetime=self.expedition.schedule.space_time_region.time_range.end_time, + # coordinates_selection_method="outside", + # ) + + #! TODO: FIX! + fieldset = copernicusmarine.FieldSet.from_copernicus( + self.expedition.schedule.space_time_region, + self.variables, + self.dimensions, + allow_time_extrapolation=self.allow_time_extrapolation, + ) + except FileNotFoundError as e: + raise FileNotFoundError( + "ERROR" # TODO: improve error message! + ) from e + + else: # from fetched data on disk + try: + data_dir = self._get_data_dir(self.directory) + joined_filepaths = { + key: data_dir.joinpath(filename) + for key, filename in self.filenames.items() + } + fieldset = FieldSet.from_netcdf( + joined_filepaths, + self.variables, + self.dimensions, + allow_time_extrapolation=self.allow_time_extrapolation, + ) + except FileNotFoundError as e: + raise FileNotFoundError( + f"Input data for instrument {self.name} not found. Have you run the `virtualship fetch` command??" + ) from e # interpolation methods for var in (v for v in self.variables if v not in ("U", "V")): From 870271bc04626a61c0568efae53874d29a0e4bd1 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 5 Nov 2025 16:53:31 +0100 Subject: [PATCH 47/97] first draft direct ingestion via copernicusmarine (CTD only) --- src/virtualship/cli/commands.py | 11 +++- src/virtualship/expedition/do_expedition.py | 25 +++++--- src/virtualship/instruments/base.py | 70 +++++++++++++-------- src/virtualship/instruments/ctd.py | 5 +- src/virtualship/models/expedition.py | 44 +++++++------ src/virtualship/utils.py | 24 ++++++- 6 files changed, 123 insertions(+), 56 deletions(-) diff --git a/src/virtualship/cli/commands.py b/src/virtualship/cli/commands.py index 3e83be3b9..0f3c058ae 100644 --- a/src/virtualship/cli/commands.py +++ b/src/virtualship/cli/commands.py @@ -106,11 +106,18 @@ def fetch(path: str | Path, username: str | None, password: str | None) -> None: _fetch(path, username, password) +# TODO: also add option to 'stream' via link to dir elsewhere, e.g. simlink or path to data stored elsewhere that isn't expedition dir! @click.command() @click.argument( "path", type=click.Path(exists=True, file_okay=False, dir_okay=True, readable=True), ) -def run(path): +@click.option( + "--from-copernicusmarine", + is_flag=True, + default=False, + help="Ingest fieldsets directly via copernicusmarine toolbox.", +) +def run(path, from_copernicusmarine: bool): """Run the expedition.""" - do_expedition(Path(path)) + do_expedition(Path(path), from_copernicusmarine) diff --git a/src/virtualship/expedition/do_expedition.py b/src/virtualship/expedition/do_expedition.py index b857b72c7..eaef2b342 100644 --- a/src/virtualship/expedition/do_expedition.py +++ b/src/virtualship/expedition/do_expedition.py @@ -31,12 +31,16 @@ external_logger = logging.getLogger("parcels.tools.loggers") external_logger.setLevel(logging.WARNING) +# copernicusmarine logger (suppress INFO messages to prevent log being flooded) +logging.getLogger("copernicusmarine").setLevel("ERROR") -def do_expedition(expedition_dir: str | Path) -> None: + +def do_expedition(expedition_dir: str | Path, from_copernicusmarine: bool) -> None: """ Perform an expedition, providing terminal feedback and file output. :param expedition_dir: The base directory for the expedition. + :param from_copernicusmarine: Whether to use direct data ingestion from Copernicus Marine. Should be determined by CLI flag. """ print("\n╔═════════════════════════════════════════════════╗") print("║ VIRTUALSHIP EXPEDITION STATUS ║") @@ -61,12 +65,15 @@ def do_expedition(expedition_dir: str | Path) -> None: print("\n---- WAYPOINT VERIFICATION ----") # verify schedule is valid - data_dir = get_existing_download( - expedition_dir, - get_space_time_region_hash(expedition.schedule.space_time_region), - ) + if from_copernicusmarine: + bathy_data_dir = None + else: + bathy_data_dir = get_existing_download( + expedition_dir, + get_space_time_region_hash(expedition.schedule.space_time_region), + ) - expedition.schedule.verify(expedition.ship_config.ship_speed_knots, data_dir) + expedition.schedule.verify(expedition.ship_config.ship_speed_knots, bathy_data_dir) # simulate the schedule schedule_results = simulate_schedule( @@ -117,7 +124,11 @@ def do_expedition(expedition_dir: str | Path) -> None: measurements = getattr(schedule_results.measurements_to_simulate, attr) # initialise instrument - instrument = instrument_class(expedition=expedition, directory=expedition_dir) + instrument = instrument_class( + expedition=expedition, + directory=expedition_dir, + from_copernicusmarine=from_copernicusmarine, + ) # run simulation instrument.run( diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 0e5ff37a3..920b8dbfa 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING import copernicusmarine +import xarray as xr from yaspin import yaspin from parcels import Field, FieldSet @@ -112,7 +113,7 @@ def __init__( allow_time_extrapolation: bool, verbose_progress: bool, bathymetry_file: str = "bathymetry.nc", - direct: bool = False, + from_copernicusmarine: bool = False, ): """Initialise instrument.""" self.name = name @@ -130,35 +131,31 @@ def __init__( self.add_bathymetry = add_bathymetry self.allow_time_extrapolation = allow_time_extrapolation self.verbose_progress = verbose_progress - self.direct = direct + self.from_copernicusmarine = from_copernicusmarine def load_input_data(self) -> FieldSet: """Load and return the input data as a FieldSet for the instrument.""" - if self.direct: # if direct ingestion from Copernicus Marine is enabled + if self.from_copernicusmarine: try: - # ds = copernicusmarine.open_dataset( - # dataset_id="PHYS_REANALYSIS_ID", - # dataset_part="default", - # minimum_longitude=self.expedition.schedule.space_time_region.spatial_range.minimum_longitude, - # maximum_longitude=self.expedition.schedule.space_time_region.spatial_range.maximum_longitude, - # minimum_latitude=self.expedition.schedule.space_time_region.spatial_range.minimum_latitude, - # maximum_latitude=self.expedition.schedule.space_time_region.spatial_range.maximum_latitude, - # variables=["uo", "vo", "so", "thetao"], - # start_datetime=self.expedition.schedule.space_time_region.time_range.start_time, - # end_datetime=self.expedition.schedule.space_time_region.time_range.end_time, - # coordinates_selection_method="outside", - # ) - - #! TODO: FIX! - fieldset = copernicusmarine.FieldSet.from_copernicus( - self.expedition.schedule.space_time_region, - self.variables, - self.dimensions, - allow_time_extrapolation=self.allow_time_extrapolation, + datasets = [] + for var in self.variables.values(): + physical = ( + True if var in ("uo", "vo", "so", "thetao") else False + ) # TODO: add more if start using new physical variables! Or more dynamic way of determining? + ds = self._get_copernicus_ds( + physical=physical, var=var + ) # user should be prompted for credentials + datasets.append(ds) + + ds_concat = xr.merge(datasets) + fieldset = FieldSet.from_xarray_dataset( + ds_concat, self.variables, self.dimensions, mesh="spherical" ) - except FileNotFoundError as e: + + except Exception as e: raise FileNotFoundError( - "ERROR" # TODO: improve error message! + f"Failed to load input data directly from Copernicus Marine for instrument '{self.name}'. " + f"Please check your credentials, network connection, and variable names. Original error: {e}" ) from e else: # from fetched data on disk @@ -176,7 +173,8 @@ def load_input_data(self) -> FieldSet: ) except FileNotFoundError as e: raise FileNotFoundError( - f"Input data for instrument {self.name} not found. Have you run the `virtualship fetch` command??" + f"Input data for instrument {self.name} not found locally. Have you run the `virtualship fetch` command?" + "Alternatively, you can use the `--from-copernicusmarine` option to ingest data directly from Copernicus Marine." ) from e # interpolation methods @@ -230,3 +228,25 @@ def _get_data_dir(self, expedition_dir: Path) -> Path: ) return data_dir + + def _get_copernicus_ds(self, physical: bool, var: str) -> xr.Dataset: + """Get Copernicus Marine dataset for direct ingestion.""" + product_id = _select_product_id( + physical=physical, + schedule_start=self.expedition.schedule.space_time_region.time_range.start_time, + schedule_end=self.expedition.schedule.space_time_region.time_range.end_time, + variable=var if not physical else None, + ) + + return copernicusmarine.open_dataset( + dataset_id=product_id, + dataset_part="default", + minimum_longitude=self.expedition.schedule.space_time_region.spatial_range.minimum_longitude, + maximum_longitude=self.expedition.schedule.space_time_region.spatial_range.maximum_longitude, + minimum_latitude=self.expedition.schedule.space_time_region.spatial_range.minimum_latitude, + maximum_latitude=self.expedition.schedule.space_time_region.spatial_range.maximum_latitude, + variables=["uo", "vo", "so", "thetao"], + start_datetime=self.expedition.schedule.space_time_region.time_range.start_time, + end_datetime=self.expedition.schedule.space_time_region.time_range.end_time, + coordinates_selection_method="outside", + ) diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 8ddb2822a..7434da2d7 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -3,8 +3,8 @@ from typing import TYPE_CHECKING, ClassVar import numpy as np -from parcels import JITParticle, ParticleSet, Variable +from parcels import JITParticle, ParticleSet, Variable from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType @@ -122,7 +122,7 @@ def get_datasets_dict(self) -> dict: class CTDInstrument(Instrument): """CTD instrument class.""" - def __init__(self, expedition, directory): + def __init__(self, expedition, directory, from_copernicusmarine): """Initialize CTDInstrument.""" filenames = { "S": f"{CTD.name}_s.nc", @@ -139,6 +139,7 @@ def __init__(self, expedition, directory): add_bathymetry=True, allow_time_extrapolation=True, verbose_progress=False, + from_copernicusmarine=from_copernicusmarine, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index af85c1c53..14d9537d6 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -12,7 +12,7 @@ from parcels import Field from virtualship.errors import InstrumentsConfigError, ScheduleError from virtualship.instruments.types import InstrumentType -from virtualship.utils import _validate_numeric_mins_to_timedelta +from virtualship.utils import _get_bathy_data, _validate_numeric_mins_to_timedelta from .location import Location from .space_time_region import SpaceTimeRegion @@ -89,7 +89,7 @@ class Schedule(pydantic.BaseModel): def verify( self, ship_speed: float, - data_dir: str | Path | None, + bathy_data_dir: str | Path | None, ignore_missing_bathymetry: bool = False, *, check_space_time_region: bool = False, @@ -132,18 +132,30 @@ def verify( # check if all waypoints are in water using bathymetry data # TODO: write test that checks that will flag when waypoint is on land!! [add to existing suite of fail .verify() tests in test_expedition.py] land_waypoints = [] - if data_dir is not None: - bathymetry_path = data_dir.joinpath("bathymetry.nc") - try: - bathymetry_field = Field.from_netcdf( - bathymetry_path, - variable=("bathymetry", "deptho"), - dimensions={"lon": "longitude", "lat": "latitude"}, - ) - except Exception as e: - raise ScheduleError( - f"Problem loading bathymetry data (used to verify waypoints are in water): {e}" - ) from e + if not ignore_missing_bathymetry: + if bathy_data_dir is None: + try: + bathymetry_field = _get_bathy_data( + self.space_time_region + ).bathymetry # via copernicusmarine + except Exception as e: + raise ScheduleError( + f"Problem loading bathymetry data (used to verify waypoints are in water) directly via copernicusmarine. \n\n original message: {e}" + ) from e + + else: + bathymetry_path = bathy_data_dir.joinpath("bathymetry.nc") + try: + bathymetry_field = Field.from_netcdf( + bathymetry_path, + variable=("bathymetry", "deptho"), + dimensions={"lon": "longitude", "lat": "latitude"}, + ) + except Exception as e: + raise ScheduleError( + f"Problem loading local bathymetry data (used to verify waypoints are in water). Have you run `virtualship fetch` command?. \n\n original message: {e}" + ) from e + for wp_i, wp in enumerate(self.waypoints): try: bathymetry_field.eval( @@ -159,10 +171,6 @@ def verify( raise ScheduleError( f"The following waypoint(s) throw(s) error(s): {['#' + str(wp_i + 1) + ' ' + str(wp) for (wp_i, wp) in land_waypoints]}\n\nINFO: They are likely on land (bathymetry data cannot be interpolated to their location(s)).\n" ) - elif not ignore_missing_bathymetry: - raise ScheduleError( - "Cannot verify waypoints are in water as bathymetry data not found. Have you run `virtualship fetch` command?" - ) # check that ship will arrive on time at each waypoint (in case no unexpected event happen) time = self.waypoints[0].time diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index 8f1fadd47..a013d37e0 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -338,8 +338,8 @@ def _select_product_id( physical: bool, schedule_start, schedule_end, - username: str, - password: str, + username: str | None = None, + password: str | None = None, variable: str | None = None, ) -> str: """Determine which copernicus product id should be selected (reanalysis, reanalysis-interim, analysis & forecast), for prescribed schedule and physical vs. BGC.""" @@ -417,3 +417,23 @@ def _start_end_in_product_timerange( np.datetime64(schedule_start) >= time_min and np.datetime64(schedule_end) <= time_max ) + + +def _get_bathy_data(space_time_region) -> FieldSet: + """Bathymetry data 'streamed' directly from Copernicus Marine.""" + ds_bathymetry = copernicusmarine.open_dataset( + dataset_id="cmems_mod_glo_phy_my_0.083deg_static", + minimum_longitude=space_time_region.spatial_range.minimum_longitude, + maximum_longitude=space_time_region.spatial_range.maximum_longitude, + minimum_latitude=space_time_region.spatial_range.minimum_latitude, + maximum_latitude=space_time_region.spatial_range.maximum_latitude, + variables=["deptho"], + start_datetime=space_time_region.time_range.start_time, + end_datetime=space_time_region.time_range.end_time, + coordinates_selection_method="outside", + ) + bathymetry_variables = {"bathymetry": "deptho"} + bathymetry_dimensions = {"lon": "longitude", "lat": "latitude"} + return FieldSet.from_xarray_dataset( + ds_bathymetry, bathymetry_variables, bathymetry_dimensions + ) From 701e45d5005ee50cc8bad166ae92f1e222d87cca Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 5 Nov 2025 17:23:57 +0100 Subject: [PATCH 48/97] refactor bathymetry data handling and add (temporary) timing for performance evaluation --- src/virtualship/cli/_plan.py | 2 +- src/virtualship/expedition/do_expedition.py | 13 +++++++++++ src/virtualship/instruments/base.py | 24 ++++++++++++++------- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/src/virtualship/cli/_plan.py b/src/virtualship/cli/_plan.py index c27cb7412..6aa6ff28e 100644 --- a/src/virtualship/cli/_plan.py +++ b/src/virtualship/cli/_plan.py @@ -1046,7 +1046,7 @@ def save_pressed(self) -> None: # verify schedule expedition_editor.expedition.schedule.verify( ship_speed_value, - data_dir=None, + bathy_data_dir=None, check_space_time_region=True, ignore_missing_bathymetry=True, ) diff --git a/src/virtualship/expedition/do_expedition.py b/src/virtualship/expedition/do_expedition.py index eaef2b342..af9fc4aad 100644 --- a/src/virtualship/expedition/do_expedition.py +++ b/src/virtualship/expedition/do_expedition.py @@ -42,6 +42,13 @@ def do_expedition(expedition_dir: str | Path, from_copernicusmarine: bool) -> No :param expedition_dir: The base directory for the expedition. :param from_copernicusmarine: Whether to use direct data ingestion from Copernicus Marine. Should be determined by CLI flag. """ + # ################################# TEMPORARY TIMER: START ################################# + import time + + start_time = time.time() + print("[TIMER] Expedition started...") + # ################################# TEMPORARY TIMER: START ################################# + print("\n╔═════════════════════════════════════════════════╗") print("║ VIRTUALSHIP EXPEDITION STATUS ║") print("╚═════════════════════════════════════════════════╝") @@ -145,6 +152,12 @@ def do_expedition(expedition_dir: str | Path, from_copernicusmarine: bool) -> No ) print("\n------------- END -------------\n") + ################################# TEMPORARY TIMER: END ################################# + end_time = time.time() + elapsed = end_time - start_time + print(f"[TIMER] Expedition completed in {elapsed:.2f} seconds.") + ################################# TEMPORARY TIMER: END ################################# + def _load_checkpoint(expedition_dir: Path) -> Checkpoint | None: file_path = expedition_dir.joinpath(CHECKPOINT) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 920b8dbfa..ed40e151f 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -9,7 +9,7 @@ from parcels import Field, FieldSet from virtualship.cli._fetch import get_existing_download, get_space_time_region_hash -from virtualship.utils import _select_product_id, ship_spinner +from virtualship.utils import _get_bathy_data, _select_product_id, ship_spinner if TYPE_CHECKING: from virtualship.models import Expedition, SpaceTimeRegion @@ -147,7 +147,7 @@ def load_input_data(self) -> FieldSet: ) # user should be prompted for credentials datasets.append(ds) - ds_concat = xr.merge(datasets) + ds_concat = xr.merge(datasets) # TODO: deal with WARNINGS? fieldset = FieldSet.from_xarray_dataset( ds_concat, self.variables, self.dimensions, mesh="spherical" ) @@ -183,16 +183,24 @@ def load_input_data(self) -> FieldSet: # depth negative for g in fieldset.gridset.grids: g.negate_depth() + # bathymetry data if self.add_bathymetry: - bathymetry_field = Field.from_netcdf( - data_dir.joinpath(self.bathymetry_file), - variable=("bathymetry", "deptho"), - dimensions={"lon": "longitude", "lat": "latitude"}, - ) + if self.from_copernicusmarine: + bathymetry_field = _get_bathy_data( + self.expedition.schedule.space_time_region + ).bathymetry + else: + bathymetry_field = Field.from_netcdf( + data_dir.joinpath(self.bathymetry_file), + variable=("bathymetry", "deptho"), + dimensions={"lon": "longitude", "lat": "latitude"}, + ) bathymetry_field.data = -bathymetry_field.data fieldset.add_field(bathymetry_field) - fieldset.computeTimeChunk(0, 1) # read in data already + + # TODO: is this line necessary?! + # fieldset.computeTimeChunk(0, 1) # read in data already return fieldset From 81941155258ca4b068bcf45eaebabf9096366a52 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 6 Nov 2025 08:58:28 +0100 Subject: [PATCH 49/97] update instrument constructors for Copernicus Marine ingestion --- src/virtualship/cli/_fetch.py | 15 ++++++++++++++- src/virtualship/instruments/adcp.py | 5 +++-- src/virtualship/instruments/argo_float.py | 5 +++-- src/virtualship/instruments/base.py | 1 + src/virtualship/instruments/ctd_bgc.py | 5 +++-- src/virtualship/instruments/drifter.py | 5 +++-- src/virtualship/instruments/ship_underwater_st.py | 5 +++-- src/virtualship/instruments/xbt.py | 5 +++-- 8 files changed, 33 insertions(+), 13 deletions(-) diff --git a/src/virtualship/cli/_fetch.py b/src/virtualship/cli/_fetch.py index a41687d00..9722fe36f 100644 --- a/src/virtualship/cli/_fetch.py +++ b/src/virtualship/cli/_fetch.py @@ -39,6 +39,13 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None be provided on prompt, via command line arguments, or via a YAML config file. Run `virtualship fetch` on an expedition for more info. """ + # ################################# TEMPORARY TIMER: START ################################# + import time + + start_time = time.time() + print("[TIMER] Expedition started...") + # ################################# TEMPORARY TIMER: START ################################# + if sum([username is None, password is None]) == 1: raise ValueError("Both username and password must be provided when using CLI.") @@ -51,7 +58,7 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None expedition.schedule.verify( expedition.ship_config.ship_speed_knots, - data_dir=None, + bathy_data_dir=None, check_space_time_region=True, ignore_missing_bathymetry=True, ) @@ -129,6 +136,12 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None complete_download(download_folder) + ################################# TEMPORARY TIMER: END ################################# + end_time = time.time() + elapsed = end_time - start_time + print(f"[TIMER] Expedition completed in {elapsed:.2f} seconds.") + ################################# TEMPORARY TIMER: END ################################# + def _hash(s: str, *, length: int) -> str: """Create a hash of a string.""" diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index 857e96555..2702cbfdd 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -2,8 +2,8 @@ from typing import ClassVar import numpy as np -from parcels import ParticleSet, ScipyParticle, Variable +from parcels import ParticleSet, ScipyParticle, Variable from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.utils import ( @@ -95,7 +95,7 @@ def get_datasets_dict(self) -> dict: class ADCPInstrument(Instrument): """ADCP instrument class.""" - def __init__(self, expedition, directory): + def __init__(self, expedition, directory, from_copernicusmarine): """Initialize ADCPInstrument.""" filenames = { "U": f"{ADCP.name}_uv.nc", @@ -111,6 +111,7 @@ def __init__(self, expedition, directory): add_bathymetry=False, allow_time_extrapolation=True, verbose_progress=False, + from_copernicusmarine=from_copernicusmarine, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index 649616052..30def124d 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -4,6 +4,7 @@ from typing import ClassVar import numpy as np + from parcels import ( AdvectionRK4, JITParticle, @@ -11,7 +12,6 @@ StatusCode, Variable, ) - from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime @@ -187,7 +187,7 @@ def get_datasets_dict(self) -> dict: class ArgoFloatInstrument(Instrument): """ArgoFloat instrument class.""" - def __init__(self, expedition, directory): + def __init__(self, expedition, directory, from_copernicusmarine): """Initialize ArgoFloatInstrument.""" filenames = { "U": f"{ArgoFloat.name}_uv.nc", @@ -205,6 +205,7 @@ def __init__(self, expedition, directory): add_bathymetry=False, allow_time_extrapolation=False, verbose_progress=True, + from_copernicusmarine=from_copernicusmarine, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index ed40e151f..a0e34e42a 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -101,6 +101,7 @@ class Instrument(abc.ABC): # TODO: update documentation/quickstart # TODO: update tests # TODO: if use direct ingestion as primary data sourcing, can substantially cut code base (including _fetch.py, InputDataset objects). Consider this for Parcels v4 transition. + #! TODO: how is this handling credentials?! Seems to work already, are these set up from my previous instances of using copernicusmarine? Therefore users will only have to do it once too? def __init__( self, diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 23c978f70..89173bbf5 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np -from parcels import JITParticle, ParticleSet, Variable +from parcels import JITParticle, ParticleSet, Variable from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime @@ -169,7 +169,7 @@ def get_datasets_dict(self) -> dict: class CTD_BGCInstrument(Instrument): """CTD_BGC instrument class.""" - def __init__(self, expedition, directory): + def __init__(self, expedition, directory, from_copernicusmarine): """Initialize CTD_BGCInstrument.""" filenames = { "o2": f"{CTD_BGC.name}_o2.nc", @@ -198,6 +198,7 @@ def __init__(self, expedition, directory): add_bathymetry=True, allow_time_extrapolation=True, verbose_progress=False, + from_copernicusmarine=from_copernicusmarine, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index f854d51f9..182100c23 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np -from parcels import AdvectionRK4, JITParticle, ParticleSet, Variable +from parcels import AdvectionRK4, JITParticle, ParticleSet, Variable from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime @@ -108,7 +108,7 @@ def get_datasets_dict(self) -> dict: class DrifterInstrument(Instrument): """Drifter instrument class.""" - def __init__(self, expedition, directory): + def __init__(self, expedition, directory, from_copernicusmarine): """Initialize DrifterInstrument.""" filenames = { "U": f"{Drifter.name}_uv.nc", @@ -125,6 +125,7 @@ def __init__(self, expedition, directory): add_bathymetry=False, allow_time_extrapolation=False, verbose_progress=True, + from_copernicusmarine=from_copernicusmarine, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index 161bb1843..5f793d3d1 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -2,8 +2,8 @@ from typing import ClassVar import numpy as np -from parcels import ParticleSet, ScipyParticle, Variable +from parcels import ParticleSet, ScipyParticle, Variable from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.utils import add_dummy_UV, register_input_dataset, register_instrument @@ -100,7 +100,7 @@ def get_datasets_dict(self) -> dict: class Underwater_STInstrument(Instrument): """Underwater_ST instrument class.""" - def __init__(self, expedition, directory): + def __init__(self, expedition, directory, from_copernicusmarine): """Initialize Underwater_STInstrument.""" filenames = { "S": f"{Underwater_ST.name}_s.nc", @@ -117,6 +117,7 @@ def __init__(self, expedition, directory): add_bathymetry=False, allow_time_extrapolation=True, verbose_progress=False, + from_copernicusmarine=from_copernicusmarine, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index 68502533c..ab11ed67e 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np -from parcels import JITParticle, ParticleSet, Variable +from parcels import JITParticle, ParticleSet, Variable from virtualship.instruments.base import InputDataset, Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime @@ -127,7 +127,7 @@ def get_datasets_dict(self) -> dict: class XBTInstrument(Instrument): """XBT instrument class.""" - def __init__(self, expedition, directory): + def __init__(self, expedition, directory, from_copernicusmarine): """Initialize XBTInstrument.""" filenames = { "U": f"{XBT.name}_uv.nc", @@ -145,6 +145,7 @@ def __init__(self, expedition, directory): add_bathymetry=True, allow_time_extrapolation=True, verbose_progress=False, + from_copernicusmarine=from_copernicusmarine, ) def simulate(self, measurements, out_path) -> None: From 446b8d0b503a395c99806f0e554501ebf3c80e72 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 6 Nov 2025 09:16:47 +0100 Subject: [PATCH 50/97] move expedition/do_expedition.py to cli/_run.py, rename Instrument.run() to Instrument.execute() --- .../do_expedition.py => cli/_run.py} | 21 +++++++++---------- src/virtualship/cli/commands.py | 4 ++-- src/virtualship/instruments/base.py | 2 +- 3 files changed, 13 insertions(+), 14 deletions(-) rename src/virtualship/{expedition/do_expedition.py => cli/_run.py} (94%) diff --git a/src/virtualship/expedition/do_expedition.py b/src/virtualship/cli/_run.py similarity index 94% rename from src/virtualship/expedition/do_expedition.py rename to src/virtualship/cli/_run.py index b857b72c7..fb67ba5d2 100644 --- a/src/virtualship/expedition/do_expedition.py +++ b/src/virtualship/cli/_run.py @@ -8,6 +8,13 @@ import pyproj from virtualship.cli._fetch import get_existing_download, get_space_time_region_hash +from virtualship.expedition.checkpoint import Checkpoint +from virtualship.expedition.expedition_cost import expedition_cost +from virtualship.expedition.simulate_schedule import ( + MeasurementsToSimulate, + ScheduleProblem, + simulate_schedule, +) from virtualship.models import Schedule from virtualship.utils import ( CHECKPOINT, @@ -15,14 +22,6 @@ get_instrument_class, ) -from .checkpoint import Checkpoint -from .expedition_cost import expedition_cost -from .simulate_schedule import ( - MeasurementsToSimulate, - ScheduleProblem, - simulate_schedule, -) - # projection used to sail between waypoints projection = pyproj.Geod(ellps="WGS84") @@ -32,7 +31,7 @@ external_logger.setLevel(logging.WARNING) -def do_expedition(expedition_dir: str | Path) -> None: +def _run(expedition_dir: str | Path) -> None: """ Perform an expedition, providing terminal feedback and file output. @@ -119,8 +118,8 @@ def do_expedition(expedition_dir: str | Path) -> None: # initialise instrument instrument = instrument_class(expedition=expedition, directory=expedition_dir) - # run simulation - instrument.run( + # execute simulation + instrument.execute( measurements=measurements, out_path=expedition_dir.joinpath("results", f"{itype.name.lower()}.zarr"), ) diff --git a/src/virtualship/cli/commands.py b/src/virtualship/cli/commands.py index 3e83be3b9..4571174df 100644 --- a/src/virtualship/cli/commands.py +++ b/src/virtualship/cli/commands.py @@ -5,7 +5,7 @@ from virtualship import utils from virtualship.cli._fetch import _fetch from virtualship.cli._plan import _plan -from virtualship.expedition.do_expedition import do_expedition +from virtualship.cli._run import _run from virtualship.utils import ( EXPEDITION, mfp_to_yaml, @@ -113,4 +113,4 @@ def fetch(path: str | Path, username: str | None, password: str | None) -> None: ) def run(path): """Run the expedition.""" - do_expedition(Path(path)) + _run(Path(path)) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 0f5307029..4497764c0 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -169,7 +169,7 @@ def load_input_data(self) -> FieldSet: def simulate(self, data_dir: Path, measurements: list, out_path: str | Path): """Simulate instrument measurements.""" - def run(self, measurements: list, out_path: str | Path) -> None: + def execute(self, measurements: list, out_path: str | Path) -> None: """Run instrument simulation.""" # TODO: this will have to be able to handle the non-spinner/instead progress bar for drifters and argos! From c97d31937a76c178ae8314c3e5ac2c367a4a3f3f Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 6 Nov 2025 09:40:00 +0100 Subject: [PATCH 51/97] move checkpoint class to models, move expedition_cost() to utils.py --- src/virtualship/cli/_run.py | 2 +- src/virtualship/expedition/expedition_cost.py | 27 ------------------- .../{expedition => models}/checkpoint.py | 0 src/virtualship/utils.py | 24 +++++++++++++++++ 4 files changed, 25 insertions(+), 28 deletions(-) delete mode 100644 src/virtualship/expedition/expedition_cost.py rename src/virtualship/{expedition => models}/checkpoint.py (100%) diff --git a/src/virtualship/cli/_run.py b/src/virtualship/cli/_run.py index fb67ba5d2..468151665 100644 --- a/src/virtualship/cli/_run.py +++ b/src/virtualship/cli/_run.py @@ -8,7 +8,6 @@ import pyproj from virtualship.cli._fetch import get_existing_download, get_space_time_region_hash -from virtualship.expedition.checkpoint import Checkpoint from virtualship.expedition.expedition_cost import expedition_cost from virtualship.expedition.simulate_schedule import ( MeasurementsToSimulate, @@ -16,6 +15,7 @@ simulate_schedule, ) from virtualship.models import Schedule +from virtualship.models.checkpoint import Checkpoint from virtualship.utils import ( CHECKPOINT, _get_expedition, diff --git a/src/virtualship/expedition/expedition_cost.py b/src/virtualship/expedition/expedition_cost.py deleted file mode 100644 index cab6ab7d0..000000000 --- a/src/virtualship/expedition/expedition_cost.py +++ /dev/null @@ -1,27 +0,0 @@ -"""expedition_cost function.""" - -from datetime import timedelta - -from .simulate_schedule import ScheduleOk - - -def expedition_cost(schedule_results: ScheduleOk, time_past: timedelta) -> float: - """ - Calculate the cost of the expedition in US$. - - :param schedule_results: Results from schedule simulation. - :param time_past: Time the expedition took. - :returns: The calculated cost of the expedition in US$. - """ - SHIP_COST_PER_DAY = 30000 - DRIFTER_DEPLOY_COST = 2500 - ARGO_DEPLOY_COST = 15000 - - ship_cost = SHIP_COST_PER_DAY / 24 * time_past.total_seconds() // 3600 - num_argos = len(schedule_results.measurements_to_simulate.argo_floats) - argo_cost = num_argos * ARGO_DEPLOY_COST - num_drifters = len(schedule_results.measurements_to_simulate.drifters) - drifter_cost = num_drifters * DRIFTER_DEPLOY_COST - - cost = ship_cost + argo_cost + drifter_cost - return cost diff --git a/src/virtualship/expedition/checkpoint.py b/src/virtualship/models/checkpoint.py similarity index 100% rename from src/virtualship/expedition/checkpoint.py rename to src/virtualship/models/checkpoint.py diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index 8f1fadd47..af98cd3d0 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -15,8 +15,10 @@ from virtualship.errors import CopernicusCatalogueError if TYPE_CHECKING: + from virtualship.expedition.simulate_schedule import ScheduleOk from virtualship.models import Expedition + import pandas as pd import yaml from pydantic import BaseModel @@ -417,3 +419,25 @@ def _start_end_in_product_timerange( np.datetime64(schedule_start) >= time_min and np.datetime64(schedule_end) <= time_max ) + + +def expedition_cost(schedule_results: ScheduleOk, time_past: timedelta) -> float: + """ + Calculate the cost of the expedition in US$. + + :param schedule_results: Results from schedule simulation. + :param time_past: Time the expedition took. + :returns: The calculated cost of the expedition in US$. + """ + SHIP_COST_PER_DAY = 30000 + DRIFTER_DEPLOY_COST = 2500 + ARGO_DEPLOY_COST = 15000 + + ship_cost = SHIP_COST_PER_DAY / 24 * time_past.total_seconds() // 3600 + num_argos = len(schedule_results.measurements_to_simulate.argo_floats) + argo_cost = num_argos * ARGO_DEPLOY_COST + num_drifters = len(schedule_results.measurements_to_simulate.drifters) + drifter_cost = num_drifters * DRIFTER_DEPLOY_COST + + cost = ship_cost + argo_cost + drifter_cost + return cost From d1acdef6d88cf91f3842e47d6022256efe50dce3 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 6 Nov 2025 09:48:40 +0100 Subject: [PATCH 52/97] update imports for expedition_cost --- src/virtualship/cli/_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/virtualship/cli/_run.py b/src/virtualship/cli/_run.py index 468151665..d364dae59 100644 --- a/src/virtualship/cli/_run.py +++ b/src/virtualship/cli/_run.py @@ -8,7 +8,6 @@ import pyproj from virtualship.cli._fetch import get_existing_download, get_space_time_region_hash -from virtualship.expedition.expedition_cost import expedition_cost from virtualship.expedition.simulate_schedule import ( MeasurementsToSimulate, ScheduleProblem, @@ -19,6 +18,7 @@ from virtualship.utils import ( CHECKPOINT, _get_expedition, + expedition_cost, get_instrument_class, ) From d7531ae416131a3e1bba0a9cf6ac9b6e75943658 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 6 Nov 2025 14:14:10 +0100 Subject: [PATCH 53/97] working with drifters (bodge), CTD_BGC not yet working --- src/virtualship/instruments/base.py | 97 ++++++++++++++++++++++---- src/virtualship/instruments/ctd_bgc.py | 2 + src/virtualship/utils.py | 4 ++ 3 files changed, 89 insertions(+), 14 deletions(-) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index a0e34e42a..9f3e9f532 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -4,12 +4,19 @@ from typing import TYPE_CHECKING import copernicusmarine +import numpy as np import xarray as xr from yaspin import yaspin from parcels import Field, FieldSet from virtualship.cli._fetch import get_existing_download, get_space_time_region_hash -from virtualship.utils import _get_bathy_data, _select_product_id, ship_spinner +from virtualship.utils import ( + COPERNICUSMARINE_BGC_VARIABLES, + COPERNICUSMARINE_PHYS_VARIABLES, + _get_bathy_data, + _select_product_id, + ship_spinner, +) if TYPE_CHECKING: from virtualship.models import Expedition, SpaceTimeRegion @@ -140,15 +147,25 @@ def load_input_data(self) -> FieldSet: try: datasets = [] for var in self.variables.values(): - physical = ( - True if var in ("uo", "vo", "so", "thetao") else False - ) # TODO: add more if start using new physical variables! Or more dynamic way of determining? - ds = self._get_copernicus_ds( - physical=physical, var=var - ) # user should be prompted for credentials + physical = True if var in COPERNICUSMARINE_PHYS_VARIABLES else False + + # TODO: TEMPORARY BODGE FOR DRIFTER INSTRUMENT - REMOVE WHEN ABLE TO! + if self.name == "Drifter": + ds = self._get_copernicus_ds_DRIFTER(physical=physical, var=var) + else: + ds = self._get_copernicus_ds(physical=physical, var=var) datasets.append(ds) + # make sure time dims are matched if BGC variables are present (different monthly/daily resolutions can impact fieldset_endtime in simulate) + if any( + key in COPERNICUSMARINE_BGC_VARIABLES + for key in ds.keys() + for ds in datasets + ): + datasets = self._align_temporal(datasets) + ds_concat = xr.merge(datasets) # TODO: deal with WARNINGS? + fieldset = FieldSet.from_xarray_dataset( ds_concat, self.variables, self.dimensions, mesh="spherical" ) @@ -200,9 +217,6 @@ def load_input_data(self) -> FieldSet: bathymetry_field.data = -bathymetry_field.data fieldset.add_field(bathymetry_field) - # TODO: is this line necessary?! - # fieldset.computeTimeChunk(0, 1) # read in data already - return fieldset @abc.abstractmethod @@ -211,8 +225,6 @@ def simulate(self, data_dir: Path, measurements: list, out_path: str | Path): def run(self, measurements: list, out_path: str | Path) -> None: """Run instrument simulation.""" - # TODO: this will have to be able to handle the non-spinner/instead progress bar for drifters and argos! - if not self.verbose_progress: with yaspin( text=f"Simulating {self.name} measurements... ", @@ -226,6 +238,8 @@ def run(self, measurements: list, out_path: str | Path) -> None: self.simulate(measurements, out_path) print("\n") + # self.simulate(measurements, out_path) + def _get_data_dir(self, expedition_dir: Path) -> Path: space_time_region_hash = get_space_time_region_hash( self.expedition.schedule.space_time_region @@ -238,7 +252,11 @@ def _get_data_dir(self, expedition_dir: Path) -> Path: return data_dir - def _get_copernicus_ds(self, physical: bool, var: str) -> xr.Dataset: + def _get_copernicus_ds( + self, + physical: bool, + var: str, + ) -> xr.Dataset: """Get Copernicus Marine dataset for direct ingestion.""" product_id = _select_product_id( physical=physical, @@ -254,8 +272,59 @@ def _get_copernicus_ds(self, physical: bool, var: str) -> xr.Dataset: maximum_longitude=self.expedition.schedule.space_time_region.spatial_range.maximum_longitude, minimum_latitude=self.expedition.schedule.space_time_region.spatial_range.minimum_latitude, maximum_latitude=self.expedition.schedule.space_time_region.spatial_range.maximum_latitude, - variables=["uo", "vo", "so", "thetao"], + variables=[var], start_datetime=self.expedition.schedule.space_time_region.time_range.start_time, end_datetime=self.expedition.schedule.space_time_region.time_range.end_time, coordinates_selection_method="outside", ) + + # TODO: TEMPORARY BODGE FOR DRIFTER INSTRUMENT - REMOVE WHEN ABLE TO! + def _get_copernicus_ds_DRIFTER( + self, + physical: bool, + var: str, + ) -> xr.Dataset: + """Get Copernicus Marine dataset for direct ingestion.""" + product_id = _select_product_id( + physical=physical, + schedule_start=self.expedition.schedule.space_time_region.time_range.start_time, + schedule_end=self.expedition.schedule.space_time_region.time_range.end_time, + variable=var if not physical else None, + ) + + return copernicusmarine.open_dataset( + dataset_id=product_id, + dataset_part="default", + minimum_longitude=self.expedition.schedule.space_time_region.spatial_range.minimum_longitude + - 3.0, + maximum_longitude=self.expedition.schedule.space_time_region.spatial_range.maximum_longitude + + 3.0, + minimum_latitude=self.expedition.schedule.space_time_region.spatial_range.minimum_latitude + - 3.0, + maximum_latitude=self.expedition.schedule.space_time_region.spatial_range.maximum_latitude + + 3.0, + maximum_depth=1.0, + minimum_depth=1.0, + variables=[var], + start_datetime=self.expedition.schedule.space_time_region.time_range.start_time, + end_datetime=self.expedition.schedule.space_time_region.time_range.end_time + + timedelta(days=21.0), + coordinates_selection_method="outside", + ) + + def _align_temporal(self, datasets: list[xr.Dataset]) -> list[xr.Dataset]: + """Align monthly and daily time dims of multiple datasets (by repeating monthly values daily).""" + reference_time = datasets[ + np.argmax(ds.time for ds in datasets) + ].time # daily timeseries + + datasets_aligned = [] + for ds in datasets: + if not np.array_equal(ds.time, reference_time): + # TODO: NEED TO CHOOSE BEST METHOD HERE + # ds = ds.resample(time="1D").ffill().reindex(time=reference_time) + # ds = ds.resample(time="1D").ffill() + ds = ds.reindex({"time": reference_time}, method="nearest") + datasets_aligned.append(ds) + + return datasets_aligned diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 89173bbf5..505e4bcc1 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -272,6 +272,8 @@ def simulate(self, measurements, out_path) -> None: # define output file for the simulation out_file = ctd_bgc_particleset.ParticleFile(name=out_path, outputdt=OUTPUT_DT) + breakpoint() + # execute simulation ctd_bgc_particleset.execute( [ diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index a013d37e0..19a6d30b3 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -333,6 +333,10 @@ def add_dummy_UV(fieldset: FieldSet): "phyc": "cmems_mod_glo_bgc_myint_0.25deg_P1M-m", } +# variables used in VirtualShip which are physical or biogeochemical variables, respectively +COPERNICUSMARINE_PHYS_VARIABLES = ["uo", "vo", "so", "thetao"] +COPERNICUSMARINE_BGC_VARIABLES = ["o2", "chl", "no3", "po4", "ph", "phyc", "nppv"] + def _select_product_id( physical: bool, From 118e685f60c0e7e8f9d42894953ce1ec707b1be8 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Fri, 7 Nov 2025 09:18:42 +0100 Subject: [PATCH 54/97] remove fetch and all associated logic --- src/virtualship/cli/_creds.py | 5 + src/virtualship/cli/_fetch.py | 240 ------------------ src/virtualship/cli/_run.py | 16 +- src/virtualship/cli/commands.py | 41 +-- src/virtualship/cli/main.py | 1 - src/virtualship/instruments/adcp.py | 46 +--- src/virtualship/instruments/argo_float.py | 57 +---- src/virtualship/instruments/base.py | 197 +++----------- src/virtualship/instruments/ctd.py | 50 +--- src/virtualship/instruments/ctd_bgc.py | 75 +----- src/virtualship/instruments/drifter.py | 52 +--- .../instruments/ship_underwater_st.py | 52 +--- src/virtualship/instruments/xbt.py | 57 +---- src/virtualship/models/expedition.py | 35 +-- src/virtualship/utils.py | 15 +- 15 files changed, 76 insertions(+), 863 deletions(-) delete mode 100644 src/virtualship/cli/_fetch.py diff --git a/src/virtualship/cli/_creds.py b/src/virtualship/cli/_creds.py index 9f1d24357..d9169ec4e 100644 --- a/src/virtualship/cli/_creds.py +++ b/src/virtualship/cli/_creds.py @@ -1,3 +1,8 @@ +# + + +# TODO: TO DELETE?! + from __future__ import annotations from pathlib import Path diff --git a/src/virtualship/cli/_fetch.py b/src/virtualship/cli/_fetch.py deleted file mode 100644 index 9722fe36f..000000000 --- a/src/virtualship/cli/_fetch.py +++ /dev/null @@ -1,240 +0,0 @@ -from __future__ import annotations - -import hashlib -import shutil -from datetime import datetime -from pathlib import Path -from typing import TYPE_CHECKING - -import copernicusmarine -from copernicusmarine.core_functions.credentials_utils import InvalidUsernameOrPassword -from pydantic import BaseModel - -from virtualship.errors import IncompleteDownloadError -from virtualship.utils import ( - _dump_yaml, - _generic_load_yaml, - _get_expedition, - get_input_dataset_class, -) - -if TYPE_CHECKING: - from virtualship.models import SpaceTimeRegion - -import click - -import virtualship.cli._creds as creds -from virtualship.utils import EXPEDITION - -DOWNLOAD_METADATA = "download_metadata.yaml" - - -def _fetch(path: str | Path, username: str | None, password: str | None) -> None: - """ - Download input data for an expedition. - - Entrypoint for the tool to download data based on space-time region provided in the - schedule file. Data is downloaded from Copernicus Marine, credentials for which can be - obtained via registration: https://data.marine.copernicus.eu/register . Credentials can - be provided on prompt, via command line arguments, or via a YAML config file. Run - `virtualship fetch` on an expedition for more info. - """ - # ################################# TEMPORARY TIMER: START ################################# - import time - - start_time = time.time() - print("[TIMER] Expedition started...") - # ################################# TEMPORARY TIMER: START ################################# - - if sum([username is None, password is None]) == 1: - raise ValueError("Both username and password must be provided when using CLI.") - - path = Path(path) - - data_dir = path / "data" - data_dir.mkdir(exist_ok=True) - - expedition = _get_expedition(path) - - expedition.schedule.verify( - expedition.ship_config.ship_speed_knots, - bathy_data_dir=None, - check_space_time_region=True, - ignore_missing_bathymetry=True, - ) - - space_time_region_hash = get_space_time_region_hash( - expedition.schedule.space_time_region - ) - - existing_download = get_existing_download(data_dir, space_time_region_hash) - if existing_download is not None: - click.echo( - f"Data download for space-time region already completed ('{existing_download}')." - ) - return - - creds_path = path / creds.CREDENTIALS_FILE - credentials = {} - credentials["username"], credentials["password"] = creds.get_credentials_flow( - username, password, creds_path - ) - - # Extract instruments and space_time_region details from expedition - instruments_in_expedition = expedition.get_instruments() - space_time_region = expedition.schedule.space_time_region - - # Create download folder and set download metadata - download_folder = data_dir / hash_to_filename(space_time_region_hash) - download_folder.mkdir() - DownloadMetadata(download_complete=False).to_yaml( - download_folder / DOWNLOAD_METADATA - ) - shutil.copyfile(path / EXPEDITION, download_folder / EXPEDITION) - - click.echo(f"\n\n{(' Fetching data for: Bathymetry ').center(80, '=')}\n\n") - - # bathymetry (for all expeditions) - copernicusmarine.subset( - dataset_id="cmems_mod_glo_phy_my_0.083deg_static", - variables=["deptho"], - minimum_longitude=space_time_region.spatial_range.minimum_longitude, - maximum_longitude=space_time_region.spatial_range.maximum_longitude, - minimum_latitude=space_time_region.spatial_range.minimum_latitude, - maximum_latitude=space_time_region.spatial_range.maximum_latitude, - start_datetime=space_time_region.time_range.start_time, - end_datetime=space_time_region.time_range.start_time, - minimum_depth=abs(space_time_region.spatial_range.minimum_depth), - maximum_depth=abs(space_time_region.spatial_range.maximum_depth), - output_filename="bathymetry.nc", - output_directory=download_folder, - username=credentials["username"], - password=credentials["password"], - overwrite=True, - coordinates_selection_method="outside", - ) - - # download, only instruments present in the expedition - for itype in instruments_in_expedition: - input_dataset_class = get_input_dataset_class(itype) - if input_dataset_class is None: - raise RuntimeError(f"No input dataset class found for type {itype}.") - click.echo( - f"\n\n{(' Fetching data for: ' + itype.value + ' ').center(80, '=')}\n\n" - ) - try: - input_dataset = input_dataset_class( - data_dir=download_folder, - credentials=credentials, - space_time_region=space_time_region, - ) - input_dataset.download_data() - except InvalidUsernameOrPassword as e: - shutil.rmtree(download_folder) - raise e - click.echo(f"{itype.value} data download completed.") - - complete_download(download_folder) - - ################################# TEMPORARY TIMER: END ################################# - end_time = time.time() - elapsed = end_time - start_time - print(f"[TIMER] Expedition completed in {elapsed:.2f} seconds.") - ################################# TEMPORARY TIMER: END ################################# - - -def _hash(s: str, *, length: int) -> str: - """Create a hash of a string.""" - assert length % 2 == 0, "Length must be even." - half_length = length // 2 - - return hashlib.shake_128(s.encode("utf-8")).hexdigest(half_length) - - -def create_hash(s: str) -> str: - """Create an 8 digit hash of a string.""" - return _hash(s, length=8) - - -def hash_model(model: BaseModel, salt: int = 0) -> str: - """ - Hash a Pydantic model. - - :param region: The region to hash. - :param salt: Salt to add to the hash. - :returns: The hash. - """ - return create_hash(model.model_dump_json() + str(salt)) - - -def get_space_time_region_hash(space_time_region: SpaceTimeRegion) -> str: - # Increment salt in the event of breaking data fetching changes with prior versions - # of virtualship where you want to force new hashes (i.e., new data downloads) - salt = 0 - return hash_model(space_time_region, salt=salt) - - -def filename_to_hash(filename: str) -> str: - """Extract hash from filename of the format YYYYMMDD_HHMMSS_{hash}.""" - parts = filename.split("_") - if len(parts) != 3: - raise ValueError( - f"Filename '{filename}' must have 3 parts delimited with underscores." - ) - return parts[-1] - - -def hash_to_filename(hash: str) -> str: - """Return a filename of the format YYYYMMDD_HHMMSS_{hash}.""" - if "_" in hash: - raise ValueError("Hash cannot contain underscores.") - return f"{datetime.now().strftime('%Y%m%d_%H%M%S')}_{hash}" - - -class DownloadMetadata(BaseModel): - """Metadata for a data download.""" - - download_complete: bool - download_date: datetime | None = None - - def to_yaml(self, file_path: str | Path) -> None: - with open(file_path, "w") as file: - _dump_yaml(self, file) - - @classmethod - def from_yaml(cls, file_path: str | Path) -> DownloadMetadata: - return _generic_load_yaml(file_path, cls) - - -def get_existing_download(data_dir: Path, space_time_region_hash: str) -> Path | None: - """Check if a download has already been completed. If so, return the path for existing download.""" - for download_path in data_dir.rglob("*"): - try: - hash = filename_to_hash(download_path.name) - except ValueError: - continue - if hash == space_time_region_hash: - assert_complete_download(download_path) - return download_path - return None - - -def assert_complete_download(download_path: Path) -> None: - download_metadata = download_path / DOWNLOAD_METADATA - try: - with open(download_metadata) as file: - assert DownloadMetadata.from_yaml(file).download_complete - except (FileNotFoundError, AssertionError) as e: - raise IncompleteDownloadError( - f"Download at {download_path} was found, but looks to be incomplete " - f"(likely due to interupting it mid-download). Please delete this folder and retry." - ) from e - return - - -def complete_download(download_path: Path) -> None: - """Mark a download as complete.""" - download_metadata = download_path / DOWNLOAD_METADATA - metadata = DownloadMetadata(download_complete=True, download_date=datetime.now()) - metadata.to_yaml(download_metadata) - return diff --git a/src/virtualship/cli/_run.py b/src/virtualship/cli/_run.py index 4eb50c664..df8a24fdb 100644 --- a/src/virtualship/cli/_run.py +++ b/src/virtualship/cli/_run.py @@ -7,7 +7,6 @@ import pyproj -from virtualship.cli._fetch import get_existing_download, get_space_time_region_hash from virtualship.expedition.simulate_schedule import ( MeasurementsToSimulate, ScheduleProblem, @@ -34,12 +33,11 @@ logging.getLogger("copernicusmarine").setLevel("ERROR") -def _run(expedition_dir: str | Path, from_copernicusmarine: bool) -> None: +def _run(expedition_dir: str | Path) -> None: """ Perform an expedition, providing terminal feedback and file output. :param expedition_dir: The base directory for the expedition. - :param from_copernicusmarine: Whether to use direct data ingestion from Copernicus Marine. Should be determined by CLI flag. """ # ################################# TEMPORARY TIMER: START ################################# import time @@ -70,16 +68,7 @@ def _run(expedition_dir: str | Path, from_copernicusmarine: bool) -> None: print("\n---- WAYPOINT VERIFICATION ----") - # verify schedule is valid - if from_copernicusmarine: - bathy_data_dir = None - else: - bathy_data_dir = get_existing_download( - expedition_dir, - get_space_time_region_hash(expedition.schedule.space_time_region), - ) - - expedition.schedule.verify(expedition.ship_config.ship_speed_knots, bathy_data_dir) + expedition.schedule.verify(expedition.ship_config.ship_speed_knots) # simulate the schedule schedule_results = simulate_schedule( @@ -133,7 +122,6 @@ def _run(expedition_dir: str | Path, from_copernicusmarine: bool) -> None: instrument = instrument_class( expedition=expedition, directory=expedition_dir, - from_copernicusmarine=from_copernicusmarine, ) # execute simulation diff --git a/src/virtualship/cli/commands.py b/src/virtualship/cli/commands.py index e2a4aaded..b17f3cbcf 100644 --- a/src/virtualship/cli/commands.py +++ b/src/virtualship/cli/commands.py @@ -3,7 +3,6 @@ import click from virtualship import utils -from virtualship.cli._fetch import _fetch from virtualship.cli._plan import _plan from virtualship.cli._run import _run from virtualship.utils import ( @@ -76,48 +75,12 @@ def plan(path): _plan(Path(path)) -@click.command() -@click.argument( - "path", - type=click.Path(exists=True, file_okay=False, dir_okay=True, readable=True), -) -@click.option( - "--username", - type=str, - default=None, - help="Copernicus Marine username.", -) -@click.option( - "--password", - type=str, - default=None, - help="Copernicus Marine password.", -) -def fetch(path: str | Path, username: str | None, password: str | None) -> None: - """ - Download input data for an expedition. - - Entrypoint for the tool to download data based on space-time region provided in the - schedule file. Data is downloaded from Copernicus Marine, credentials for which can be - obtained via registration: https://data.marine.copernicus.eu/register . Credentials can - be provided on prompt, via command line arguments, or via a YAML config file. Run - `virtualship fetch` on a expedition for more info. - """ - _fetch(path, username, password) - - # TODO: also add option to 'stream' via link to dir elsewhere, e.g. simlink or path to data stored elsewhere that isn't expedition dir! @click.command() @click.argument( "path", type=click.Path(exists=True, file_okay=False, dir_okay=True, readable=True), ) -@click.option( - "--from-copernicusmarine", - is_flag=True, - default=False, - help="Ingest fieldsets directly via copernicusmarine toolbox.", -) -def run(path, from_copernicusmarine: bool): +def run(path): """Run the expedition.""" - _run(Path(path), from_copernicusmarine) + _run(Path(path)) diff --git a/src/virtualship/cli/main.py b/src/virtualship/cli/main.py index 6ee12effc..a02a5ffb5 100644 --- a/src/virtualship/cli/main.py +++ b/src/virtualship/cli/main.py @@ -11,7 +11,6 @@ def cli(): cli.add_command(commands.init) cli.add_command(commands.plan) -cli.add_command(commands.fetch) cli.add_command(commands.run) if __name__ == "__main__": diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index 2702cbfdd..d01b675b5 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -4,10 +4,9 @@ import numpy as np from parcels import ParticleSet, ScipyParticle, Variable -from virtualship.instruments.base import InputDataset, Instrument +from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.utils import ( - register_input_dataset, register_instrument, ) @@ -46,46 +45,6 @@ def _sample_velocity(particle, fieldset, time): ) -# ===================================================== -# SECTION: InputDataset Class -# ===================================================== - - -@register_input_dataset(InstrumentType.ADCP) -class ADCPInputDataset(InputDataset): - """Input dataset for ADCP instrument.""" - - DOWNLOAD_BUFFERS: ClassVar[dict] = { - "latlon_degrees": 0.0, - "days": 0.0, - } # ADCP data requires no buffers - - DOWNLOAD_LIMITS: ClassVar[dict] = {"min_depth": 1} - - def __init__(self, data_dir, credentials, space_time_region): - """Initialise with instrument's name.""" - super().__init__( - ADCP.name, - self.DOWNLOAD_BUFFERS["latlon_degrees"], - self.DOWNLOAD_BUFFERS["days"], - space_time_region.spatial_range.minimum_depth, - space_time_region.spatial_range.maximum_depth, - data_dir, - credentials, - space_time_region, - ) - - def get_datasets_dict(self) -> dict: - """Get variable specific args for instrument.""" - return { - "UVdata": { - "physical": True, - "variables": ["uo", "vo"], - "output_filename": f"{self.name}_uv.nc", - }, - } - - # ===================================================== # SECTION: Instrument Class # ===================================================== @@ -95,7 +54,7 @@ def get_datasets_dict(self) -> dict: class ADCPInstrument(Instrument): """ADCP instrument class.""" - def __init__(self, expedition, directory, from_copernicusmarine): + def __init__(self, expedition, directory): """Initialize ADCPInstrument.""" filenames = { "U": f"{ADCP.name}_uv.nc", @@ -111,7 +70,6 @@ def __init__(self, expedition, directory, from_copernicusmarine): add_bathymetry=False, allow_time_extrapolation=True, verbose_progress=False, - from_copernicusmarine=from_copernicusmarine, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index 30def124d..ce539d6bc 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -12,10 +12,10 @@ StatusCode, Variable, ) -from virtualship.instruments.base import InputDataset, Instrument +from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime -from virtualship.utils import register_input_dataset, register_instrument +from virtualship.utils import register_instrument # ===================================================== # SECTION: Dataclass @@ -128,56 +128,6 @@ def _check_error(particle, fieldset, time): particle.delete() -# ===================================================== -# SECTION: InputDataset Class -# ===================================================== - - -@register_input_dataset(InstrumentType.ARGO_FLOAT) -class ArgoFloatInputDataset(InputDataset): - """Input dataset for ArgoFloat instrument.""" - - DOWNLOAD_BUFFERS: ClassVar[dict] = { - "latlon_degrees": 3.0, - "days": 21.0, - } - - DOWNLOAD_LIMITS: ClassVar[dict] = {"min_depth": 1} - - def __init__(self, data_dir, credentials, space_time_region): - """Initialise with instrument's name.""" - super().__init__( - ArgoFloat.name, - self.DOWNLOAD_BUFFERS["latlon_degrees"], - self.DOWNLOAD_BUFFERS["days"], - self.DOWNLOAD_LIMITS["min_depth"], - space_time_region.spatial_range.maximum_depth, - data_dir, - credentials, - space_time_region, - ) - - def get_datasets_dict(self) -> dict: - """Get variable specific args for instrument.""" - return { - "UVdata": { - "physical": True, - "variables": ["uo", "vo"], - "output_filename": f"{self.name}_uv.nc", - }, - "Sdata": { - "physical": True, - "variables": ["so"], - "output_filename": f"{self.name}_s.nc", - }, - "Tdata": { - "physical": True, - "variables": ["thetao"], - "output_filename": f"{self.name}_t.nc", - }, - } - - # ===================================================== # SECTION: Instrument Class # ===================================================== @@ -187,7 +137,7 @@ def get_datasets_dict(self) -> dict: class ArgoFloatInstrument(Instrument): """ArgoFloat instrument class.""" - def __init__(self, expedition, directory, from_copernicusmarine): + def __init__(self, expedition, directory): """Initialize ArgoFloatInstrument.""" filenames = { "U": f"{ArgoFloat.name}_uv.nc", @@ -205,7 +155,6 @@ def __init__(self, expedition, directory, from_copernicusmarine): add_bathymetry=False, allow_time_extrapolation=False, verbose_progress=True, - from_copernicusmarine=from_copernicusmarine, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index b040c3dba..311d83f61 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -8,8 +8,7 @@ import xarray as xr from yaspin import yaspin -from parcels import Field, FieldSet -from virtualship.cli._fetch import get_existing_download, get_space_time_region_hash +from parcels import FieldSet from virtualship.utils import ( COPERNICUSMARINE_BGC_VARIABLES, COPERNICUSMARINE_PHYS_VARIABLES, @@ -19,86 +18,7 @@ ) if TYPE_CHECKING: - from virtualship.models import Expedition, SpaceTimeRegion - - -class InputDataset(abc.ABC): - """Base class for instrument input datasets.""" - - # TODO: data download is performed per instrument (in `fetch`), which is a bit inefficient when some instruments can share dataa. - # TODO: However, future changes, with Parcels-v4 and copernicusmarine direct ingestion, will hopefully remove the need for fetch. - - def __init__( - self, - name: str, - latlon_buffer: float, - datetime_buffer: float, - min_depth: float, - max_depth: float, - data_dir: str, - credentials: dict, - space_time_region: "SpaceTimeRegion", - ): - """Initialise input dataset.""" - self.name = name - self.latlon_buffer = latlon_buffer - self.datetime_buffer = datetime_buffer - self.min_depth = min_depth - self.max_depth = max_depth - self.data_dir = data_dir - self.credentials = credentials - self.space_time_region = space_time_region - - @abc.abstractmethod - def get_datasets_dict(self) -> dict: - """Get parameters for instrument's variable(s) specific data download.""" - - def download_data(self) -> None: - """Download data for the instrument using copernicusmarine, with correct product ID selection.""" - parameter_args = dict( - minimum_longitude=self.space_time_region.spatial_range.minimum_longitude - - self.latlon_buffer, - maximum_longitude=self.space_time_region.spatial_range.maximum_longitude - + self.latlon_buffer, - minimum_latitude=self.space_time_region.spatial_range.minimum_latitude - - self.latlon_buffer, - maximum_latitude=self.space_time_region.spatial_range.maximum_latitude - + self.latlon_buffer, - start_datetime=self.space_time_region.time_range.start_time, - end_datetime=self.space_time_region.time_range.end_time - + timedelta(days=self.datetime_buffer), - minimum_depth=abs(self.min_depth), - maximum_depth=abs(self.max_depth), - output_directory=self.data_dir, - username=self.credentials["username"], - password=self.credentials["password"], - overwrite=True, - coordinates_selection_method="outside", - ) - - datasets_args = self.get_datasets_dict() - - for dataset in datasets_args.values(): - physical = dataset.get("physical") - if physical: - variable = None - else: - variable = dataset.get("variables")[0] # BGC variables, special case - - dataset_id = _select_product_id( - physical=physical, - schedule_start=self.space_time_region.time_range.start_time, - schedule_end=self.space_time_region.time_range.end_time, - username=self.credentials["username"], - password=self.credentials["password"], - variable=variable, - ) - download_args = { - **parameter_args, - **{k: v for k, v in dataset.items() if k != "physical"}, - "dataset_id": dataset_id, - } - copernicusmarine.subset(**download_args) + from virtualship.models import Expedition class Instrument(abc.ABC): @@ -107,7 +27,6 @@ class Instrument(abc.ABC): #! TODO List: # TODO: update documentation/quickstart # TODO: update tests - # TODO: if use direct ingestion as primary data sourcing, can substantially cut code base (including _fetch.py, InputDataset objects). Consider this for Parcels v4 transition. #! TODO: how is this handling credentials?! Seems to work already, are these set up from my previous instances of using copernicusmarine? Therefore users will only have to do it once too? def __init__( @@ -121,7 +40,6 @@ def __init__( allow_time_extrapolation: bool, verbose_progress: bool, bathymetry_file: str = "bathymetry.nc", - from_copernicusmarine: bool = False, ): """Initialise instrument.""" self.name = name @@ -139,61 +57,40 @@ def __init__( self.add_bathymetry = add_bathymetry self.allow_time_extrapolation = allow_time_extrapolation self.verbose_progress = verbose_progress - self.from_copernicusmarine = from_copernicusmarine def load_input_data(self) -> FieldSet: """Load and return the input data as a FieldSet for the instrument.""" - if self.from_copernicusmarine: - try: - datasets = [] - for var in self.variables.values(): - physical = True if var in COPERNICUSMARINE_PHYS_VARIABLES else False - - # TODO: TEMPORARY BODGE FOR DRIFTER INSTRUMENT - REMOVE WHEN ABLE TO! - if self.name == "Drifter": - ds = self._get_copernicus_ds_DRIFTER(physical=physical, var=var) - else: - ds = self._get_copernicus_ds(physical=physical, var=var) - datasets.append(ds) - - # make sure time dims are matched if BGC variables are present (different monthly/daily resolutions can impact fieldset_endtime in simulate) - if any( - key in COPERNICUSMARINE_BGC_VARIABLES - for key in ds.keys() - for ds in datasets - ): - datasets = self._align_temporal(datasets) - - ds_concat = xr.merge(datasets) # TODO: deal with WARNINGS? - - fieldset = FieldSet.from_xarray_dataset( - ds_concat, self.variables, self.dimensions, mesh="spherical" - ) - - except Exception as e: - raise FileNotFoundError( - f"Failed to load input data directly from Copernicus Marine for instrument '{self.name}'. " - f"Please check your credentials, network connection, and variable names. Original error: {e}" - ) from e + try: + datasets = [] + for var in self.variables.values(): + physical = True if var in COPERNICUSMARINE_PHYS_VARIABLES else False + + # TODO: TEMPORARY BODGE FOR DRIFTER INSTRUMENT - REMOVE WHEN ABLE TO! + if self.name == "Drifter": + ds = self._get_copernicus_ds_DRIFTER(physical=physical, var=var) + else: + ds = self._get_copernicus_ds(physical=physical, var=var) + datasets.append(ds) + + # make sure time dims are matched if BGC variables are present (different monthly/daily resolutions can impact fieldset_endtime in simulate) + if any( + key in COPERNICUSMARINE_BGC_VARIABLES + for key in ds.keys() + for ds in datasets + ): + datasets = self._align_temporal(datasets) + + ds_concat = xr.merge(datasets) # TODO: deal with WARNINGS? + + fieldset = FieldSet.from_xarray_dataset( + ds_concat, self.variables, self.dimensions, mesh="spherical" + ) - else: # from fetched data on disk - try: - data_dir = self._get_data_dir(self.directory) - joined_filepaths = { - key: data_dir.joinpath(filename) - for key, filename in self.filenames.items() - } - fieldset = FieldSet.from_netcdf( - joined_filepaths, - self.variables, - self.dimensions, - allow_time_extrapolation=self.allow_time_extrapolation, - ) - except FileNotFoundError as e: - raise FileNotFoundError( - f"Input data for instrument {self.name} not found locally. Have you run the `virtualship fetch` command?" - "Alternatively, you can use the `--from-copernicusmarine` option to ingest data directly from Copernicus Marine." - ) from e + except Exception as e: + raise FileNotFoundError( + f"Failed to load input data directly from Copernicus Marine for instrument '{self.name}'. " + f"Please check your credentials, network connection, and variable names. Original error: {e}" + ) from e # interpolation methods for var in (v for v in self.variables if v not in ("U", "V")): @@ -203,19 +100,11 @@ def load_input_data(self) -> FieldSet: g.negate_depth() # bathymetry data - if self.add_bathymetry: - if self.from_copernicusmarine: - bathymetry_field = _get_bathy_data( - self.expedition.schedule.space_time_region - ).bathymetry - else: - bathymetry_field = Field.from_netcdf( - data_dir.joinpath(self.bathymetry_file), - variable=("bathymetry", "deptho"), - dimensions={"lon": "longitude", "lat": "latitude"}, - ) - bathymetry_field.data = -bathymetry_field.data - fieldset.add_field(bathymetry_field) + bathymetry_field = _get_bathy_data( + self.expedition.schedule.space_time_region + ).bathymetry + bathymetry_field.data = -bathymetry_field.data + fieldset.add_field(bathymetry_field) return fieldset @@ -240,18 +129,6 @@ def execute(self, measurements: list, out_path: str | Path) -> None: # self.simulate(measurements, out_path) - def _get_data_dir(self, expedition_dir: Path) -> Path: - space_time_region_hash = get_space_time_region_hash( - self.expedition.schedule.space_time_region - ) - data_dir = get_existing_download(expedition_dir, space_time_region_hash) - - assert data_dir is not None, ( - "Input data hasn't been found. Have you run the `virtualship fetch` command?" - ) - - return data_dir - def _get_copernicus_ds( self, physical: bool, diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 7434da2d7..d205795b0 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -5,12 +5,12 @@ import numpy as np from parcels import JITParticle, ParticleSet, Variable -from virtualship.instruments.base import InputDataset, Instrument +from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType if TYPE_CHECKING: from virtualship.models.spacetime import Spacetime -from virtualship.utils import add_dummy_UV, register_input_dataset, register_instrument +from virtualship.utils import add_dummy_UV, register_instrument # ===================================================== # SECTION: Dataclass @@ -70,49 +70,6 @@ def _ctd_cast(particle, fieldset, time): particle.delete() -# ===================================================== -# SECTION: InputDataset Class -# ===================================================== - - -@register_input_dataset(InstrumentType.CTD) -class CTDInputDataset(InputDataset): - """Input dataset for CTD instrument.""" - - DOWNLOAD_BUFFERS: ClassVar[dict] = { - "latlon_degrees": 0.0, - "days": 0.0, - } # CTD data requires no buffers - - def __init__(self, data_dir, credentials, space_time_region): - """Initialise with instrument's name.""" - super().__init__( - CTD.name, - self.DOWNLOAD_BUFFERS["latlon_degrees"], - self.DOWNLOAD_BUFFERS["days"], - space_time_region.spatial_range.minimum_depth, - space_time_region.spatial_range.maximum_depth, - data_dir, - credentials, - space_time_region, - ) - - def get_datasets_dict(self) -> dict: - """Get variable specific args for instrument.""" - return { - "Sdata": { - "physical": True, - "variables": ["so"], - "output_filename": f"{self.name}_s.nc", - }, - "Tdata": { - "physical": True, - "variables": ["thetao"], - "output_filename": f"{self.name}_t.nc", - }, - } - - # ===================================================== # SECTION: Instrument Class # ===================================================== @@ -122,7 +79,7 @@ def get_datasets_dict(self) -> dict: class CTDInstrument(Instrument): """CTD instrument class.""" - def __init__(self, expedition, directory, from_copernicusmarine): + def __init__(self, expedition, directory): """Initialize CTDInstrument.""" filenames = { "S": f"{CTD.name}_s.nc", @@ -139,7 +96,6 @@ def __init__(self, expedition, directory, from_copernicusmarine): add_bathymetry=True, allow_time_extrapolation=True, verbose_progress=False, - from_copernicusmarine=from_copernicusmarine, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 505e4bcc1..182d274c4 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -5,10 +5,10 @@ import numpy as np from parcels import JITParticle, ParticleSet, Variable -from virtualship.instruments.base import InputDataset, Instrument +from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime -from virtualship.utils import add_dummy_UV, register_input_dataset, register_instrument +from virtualship.utils import add_dummy_UV, register_instrument # ===================================================== # SECTION: Dataclass @@ -92,74 +92,6 @@ def _ctd_bgc_cast(particle, fieldset, time): particle.delete() -# ===================================================== -# SECTION: InputDataset Class -# ===================================================== - - -@register_input_dataset(InstrumentType.CTD_BGC) -class CTD_BGCInputDataset(InputDataset): - """Input dataset object for CTD_BGC instrument.""" - - DOWNLOAD_BUFFERS: ClassVar[dict] = { - "latlon_degrees": 0.0, - "days": 0.0, - } # CTD_BGC data requires no buffers - - def __init__(self, data_dir, credentials, space_time_region): - """Initialise with instrument's name.""" - super().__init__( - CTD_BGC.name, - self.DOWNLOAD_BUFFERS["latlon_degrees"], - self.DOWNLOAD_BUFFERS["days"], - space_time_region.spatial_range.minimum_depth, - space_time_region.spatial_range.maximum_depth, - data_dir, - credentials, - space_time_region, - ) - - def get_datasets_dict(self) -> dict: - """Variable specific args for instrument.""" - return { - "o2data": { - "physical": False, - "variables": ["o2"], - "output_filename": f"{self.name}_o2.nc", - }, - "chlorodata": { - "physical": False, - "variables": ["chl"], - "output_filename": f"{self.name}_chl.nc", - }, - "nitratedata": { - "physical": False, - "variables": ["no3"], - "output_filename": f"{self.name}_no3.nc", - }, - "phosphatedata": { - "physical": False, - "variables": ["po4"], - "output_filename": f"{self.name}_po4.nc", - }, - "phdata": { - "physical": False, - "variables": ["ph"], - "output_filename": f"{self.name}_ph.nc", - }, - "phytoplanktondata": { - "physical": False, - "variables": ["phyc"], - "output_filename": f"{self.name}_phyc.nc", - }, - "primaryproductiondata": { - "physical": False, - "variables": ["nppv"], - "output_filename": f"{self.name}_nppv.nc", - }, - } - - # ===================================================== # SECTION: Instrument Class # ===================================================== @@ -169,7 +101,7 @@ def get_datasets_dict(self) -> dict: class CTD_BGCInstrument(Instrument): """CTD_BGC instrument class.""" - def __init__(self, expedition, directory, from_copernicusmarine): + def __init__(self, expedition, directory): """Initialize CTD_BGCInstrument.""" filenames = { "o2": f"{CTD_BGC.name}_o2.nc", @@ -198,7 +130,6 @@ def __init__(self, expedition, directory, from_copernicusmarine): add_bathymetry=True, allow_time_extrapolation=True, verbose_progress=False, - from_copernicusmarine=from_copernicusmarine, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index 182100c23..06f6b71f7 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -5,10 +5,10 @@ import numpy as np from parcels import AdvectionRK4, JITParticle, ParticleSet, Variable -from virtualship.instruments.base import InputDataset, Instrument +from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime -from virtualship.utils import register_input_dataset, register_instrument +from virtualship.utils import register_instrument # ===================================================== # SECTION: Dataclass @@ -54,51 +54,6 @@ def _check_lifetime(particle, fieldset, time): particle.delete() -# ===================================================== -# SECTION: InputDataset Class -# ===================================================== - - -@register_input_dataset(InstrumentType.DRIFTER) -class DrifterInputDataset(InputDataset): - """Input dataset for Drifter instrument.""" - - DOWNLOAD_BUFFERS: ClassVar[dict] = { - "latlon_degrees": 3.0, - "days": 21.0, - } - - DOWNLOAD_LIMITS: ClassVar[dict] = {"min_depth": 1, "max_depth": 1} - - def __init__(self, data_dir, credentials, space_time_region): - """Initialise with instrument's name.""" - super().__init__( - Drifter.name, - self.DOWNLOAD_BUFFERS["latlon_degrees"], - self.DOWNLOAD_BUFFERS["days"], - self.DOWNLOAD_LIMITS["min_depth"], - self.DOWNLOAD_LIMITS["max_depth"], - data_dir, - credentials, - space_time_region, - ) - - def get_datasets_dict(self) -> dict: - """Get variable specific args for instrument.""" - return { - "UVdata": { - "physical": True, - "variables": ["uo", "vo"], - "output_filename": f"{self.name}_uv.nc", - }, - "Tdata": { - "physical": True, - "variables": ["thetao"], - "output_filename": f"{self.name}_t.nc", - }, - } - - # ===================================================== # SECTION: Instrument Class # ===================================================== @@ -108,7 +63,7 @@ def get_datasets_dict(self) -> dict: class DrifterInstrument(Instrument): """Drifter instrument class.""" - def __init__(self, expedition, directory, from_copernicusmarine): + def __init__(self, expedition, directory): """Initialize DrifterInstrument.""" filenames = { "U": f"{Drifter.name}_uv.nc", @@ -125,7 +80,6 @@ def __init__(self, expedition, directory, from_copernicusmarine): add_bathymetry=False, allow_time_extrapolation=False, verbose_progress=True, - from_copernicusmarine=from_copernicusmarine, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index 5f793d3d1..f6099869d 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -4,9 +4,9 @@ import numpy as np from parcels import ParticleSet, ScipyParticle, Variable -from virtualship.instruments.base import InputDataset, Instrument +from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType -from virtualship.utils import add_dummy_UV, register_input_dataset, register_instrument +from virtualship.utils import add_dummy_UV, register_instrument # ===================================================== # SECTION: Dataclass @@ -46,51 +46,6 @@ def _sample_temperature(particle, fieldset, time): particle.T = fieldset.T[time, particle.depth, particle.lat, particle.lon] -# ===================================================== -# SECTION: InputDataset Class -# ===================================================== - - -@register_input_dataset(InstrumentType.UNDERWATER_ST) -class Underwater_STInputDataset(InputDataset): - """Input dataset for Underwater_ST instrument.""" - - DOWNLOAD_BUFFERS: ClassVar[dict] = { - "latlon_degrees": 0.0, - "days": 0.0, - } # Underwater_ST data requires no buffers - - DOWNLOAD_LIMITS: ClassVar[dict] = {"min_depth": 1} - - def __init__(self, data_dir, credentials, space_time_region): - """Initialise with instrument's name.""" - super().__init__( - Underwater_ST.name, - self.DOWNLOAD_BUFFERS["latlon_degrees"], - self.DOWNLOAD_BUFFERS["days"], - -2.0, # is always at 2m depth - -2.0, # is always at 2m depth - data_dir, - credentials, - space_time_region, - ) - - def get_datasets_dict(self) -> dict: - """Get variable specific args for instrument.""" - return { - "Sdata": { - "physical": True, - "variables": ["so"], - "output_filename": f"{self.name}_s.nc", - }, - "Tdata": { - "physical": True, - "variables": ["thetao"], - "output_filename": f"{self.name}_t.nc", - }, - } - - # ===================================================== # SECTION: Instrument Class # ===================================================== @@ -100,7 +55,7 @@ def get_datasets_dict(self) -> dict: class Underwater_STInstrument(Instrument): """Underwater_ST instrument class.""" - def __init__(self, expedition, directory, from_copernicusmarine): + def __init__(self, expedition, directory): """Initialize Underwater_STInstrument.""" filenames = { "S": f"{Underwater_ST.name}_s.nc", @@ -117,7 +72,6 @@ def __init__(self, expedition, directory, from_copernicusmarine): add_bathymetry=False, allow_time_extrapolation=True, verbose_progress=False, - from_copernicusmarine=from_copernicusmarine, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index ab11ed67e..fd88240d4 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -5,10 +5,10 @@ import numpy as np from parcels import JITParticle, ParticleSet, Variable -from virtualship.instruments.base import InputDataset, Instrument +from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime -from virtualship.utils import register_input_dataset, register_instrument +from virtualship.utils import register_instrument # ===================================================== # SECTION: Dataclass @@ -68,56 +68,6 @@ def _xbt_cast(particle, fieldset, time): particle_ddepth = particle.max_depth - particle.depth -# ===================================================== -# SECTION: InputDataset Class -# ===================================================== - - -@register_input_dataset(InstrumentType.XBT) -class XBTInputDataset(InputDataset): - """Input dataset for XBT instrument.""" - - DOWNLOAD_BUFFERS: ClassVar[dict] = { - "latlon_degrees": 3.0, - "days": 21.0, - } - - DOWNLOAD_LIMITS: ClassVar[dict] = {"min_depth": 1} - - def __init__(self, data_dir, credentials, space_time_region): - """Initialise with instrument's name.""" - super().__init__( - XBT.name, - self.DOWNLOAD_BUFFERS["latlon_degrees"], - self.DOWNLOAD_BUFFERS["days"], - self.DOWNLOAD_LIMITS["min_depth"], - space_time_region.spatial_range.maximum_depth, - data_dir, - credentials, - space_time_region, - ) - - def get_datasets_dict(self) -> dict: - """Get variable specific args for instrument.""" - return { - "UVdata": { - "physical": True, - "variables": ["uo", "vo"], - "output_filename": f"{self.name}_uv.nc", - }, - "Sdata": { - "physical": True, - "variables": ["so"], - "output_filename": f"{self.name}_s.nc", - }, - "Tdata": { - "physical": True, - "variables": ["thetao"], - "output_filename": f"{self.name}_t.nc", - }, - } - - # ===================================================== # SECTION: Instrument Class # ===================================================== @@ -127,7 +77,7 @@ def get_datasets_dict(self) -> dict: class XBTInstrument(Instrument): """XBT instrument class.""" - def __init__(self, expedition, directory, from_copernicusmarine): + def __init__(self, expedition, directory): """Initialize XBTInstrument.""" filenames = { "U": f"{XBT.name}_uv.nc", @@ -145,7 +95,6 @@ def __init__(self, expedition, directory, from_copernicusmarine): add_bathymetry=True, allow_time_extrapolation=True, verbose_progress=False, - from_copernicusmarine=from_copernicusmarine, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index 14d9537d6..aa310a97c 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -2,14 +2,12 @@ import itertools from datetime import datetime, timedelta -from pathlib import Path from typing import TYPE_CHECKING import pydantic import pyproj import yaml -from parcels import Field from virtualship.errors import InstrumentsConfigError, ScheduleError from virtualship.instruments.types import InstrumentType from virtualship.utils import _get_bathy_data, _validate_numeric_mins_to_timedelta @@ -89,7 +87,6 @@ class Schedule(pydantic.BaseModel): def verify( self, ship_speed: float, - bathy_data_dir: str | Path | None, ignore_missing_bathymetry: bool = False, *, check_space_time_region: bool = False, @@ -108,7 +105,7 @@ def verify( if check_space_time_region and self.space_time_region is None: raise ScheduleError( - "space_time_region not found in schedule, please define it to fetch the data." + "space_time_region not found in schedule, please define it to proceed." ) if len(self.waypoints) == 0: @@ -133,28 +130,14 @@ def verify( # TODO: write test that checks that will flag when waypoint is on land!! [add to existing suite of fail .verify() tests in test_expedition.py] land_waypoints = [] if not ignore_missing_bathymetry: - if bathy_data_dir is None: - try: - bathymetry_field = _get_bathy_data( - self.space_time_region - ).bathymetry # via copernicusmarine - except Exception as e: - raise ScheduleError( - f"Problem loading bathymetry data (used to verify waypoints are in water) directly via copernicusmarine. \n\n original message: {e}" - ) from e - - else: - bathymetry_path = bathy_data_dir.joinpath("bathymetry.nc") - try: - bathymetry_field = Field.from_netcdf( - bathymetry_path, - variable=("bathymetry", "deptho"), - dimensions={"lon": "longitude", "lat": "latitude"}, - ) - except Exception as e: - raise ScheduleError( - f"Problem loading local bathymetry data (used to verify waypoints are in water). Have you run `virtualship fetch` command?. \n\n original message: {e}" - ) from e + try: + bathymetry_field = _get_bathy_data( + self.space_time_region + ).bathymetry # via copernicusmarine + except Exception as e: + raise ScheduleError( + f"Problem loading bathymetry data (used to verify waypoints are in water) directly via copernicusmarine. \n\n original message: {e}" + ) from e for wp_i, wp in enumerate(self.waypoints): try: diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index dc46ded71..4ea17c4a5 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -249,19 +249,10 @@ def _get_expedition(expedition_dir: Path) -> Expedition: ) -# InstrumentType -> InputDataset and Instrument registry and registration utilities. -INPUT_DATASET_MAP = {} +# InstrumentType -> Instrument registry and registration utilities. INSTRUMENT_CLASS_MAP = {} -def register_input_dataset(instrument_type): - def decorator(cls): - INPUT_DATASET_MAP[instrument_type] = cls - return cls - - return decorator - - def register_instrument(instrument_type): def decorator(cls): INSTRUMENT_CLASS_MAP[instrument_type] = cls @@ -270,10 +261,6 @@ def decorator(cls): return decorator -def get_input_dataset_class(instrument_type): - return INPUT_DATASET_MAP.get(instrument_type) - - def get_instrument_class(instrument_type): return INSTRUMENT_CLASS_MAP.get(instrument_type) From 05686f2513229286bff5cbf3bee3789353dcad2a Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Fri, 7 Nov 2025 09:24:21 +0100 Subject: [PATCH 55/97] update docstrings/--help info --- src/virtualship/cli/commands.py | 4 ++-- src/virtualship/expedition/__init__.py | 8 +------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/src/virtualship/cli/commands.py b/src/virtualship/cli/commands.py index b17f3cbcf..2b0ce8e93 100644 --- a/src/virtualship/cli/commands.py +++ b/src/virtualship/cli/commands.py @@ -68,7 +68,7 @@ def init(path, from_mfp): ) def plan(path): """ - Launch UI to help build schedule and ship config files. + Launch UI to help build expedition configuration (YAML) file. Should you encounter any issues with using this tool, please report an issue describing the problem to the VirtualShip issue tracker at: https://github.com/OceanParcels/virtualship/issues" """ @@ -82,5 +82,5 @@ def plan(path): type=click.Path(exists=True, file_okay=False, dir_okay=True, readable=True), ) def run(path): - """Run the expedition.""" + """Execute the expedition simulations.""" _run(Path(path)) diff --git a/src/virtualship/expedition/__init__.py b/src/virtualship/expedition/__init__.py index dfa610283..7f072bbfb 100644 --- a/src/virtualship/expedition/__init__.py +++ b/src/virtualship/expedition/__init__.py @@ -1,7 +1 @@ -"""Everything for simulating an expedition.""" - -from .do_expedition import do_expedition - -__all__ = [ - "do_expedition", -] +"""Simulating an expedition.""" From 10e68b383a640038804d8effe3c22129d1f85c27 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Fri, 7 Nov 2025 11:40:19 +0100 Subject: [PATCH 56/97] add buffers to drifters and argos, plus depth limits for drifters --- src/virtualship/instruments/adcp.py | 3 + src/virtualship/instruments/argo_float.py | 7 ++ src/virtualship/instruments/base.py | 85 ++++++------------- src/virtualship/instruments/ctd.py | 2 + src/virtualship/instruments/ctd_bgc.py | 2 + src/virtualship/instruments/drifter.py | 13 ++- .../instruments/ship_underwater_st.py | 2 + src/virtualship/instruments/xbt.py | 2 + src/virtualship/models/expedition.py | 5 +- src/virtualship/utils.py | 14 +-- 10 files changed, 68 insertions(+), 67 deletions(-) diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index d01b675b5..dd675b7ee 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -61,6 +61,7 @@ def __init__(self, expedition, directory): "V": f"{ADCP.name}_uv.nc", } variables = {"U": "uo", "V": "vo"} + super().__init__( ADCP.name, expedition, @@ -70,6 +71,8 @@ def __init__(self, expedition, directory): add_bathymetry=False, allow_time_extrapolation=True, verbose_progress=False, + buffer_spec=None, + limit_spec=None, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index ce539d6bc..12ee8945a 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -146,6 +146,11 @@ def __init__(self, expedition, directory): "T": f"{ArgoFloat.name}_t.nc", } variables = {"U": "uo", "V": "vo", "S": "so", "T": "thetao"} + buffer_spec = { + "latlon": 6.0, # [degrees] + "time": 21.0, # [days] + } + super().__init__( ArgoFloat.name, expedition, @@ -155,6 +160,8 @@ def __init__(self, expedition, directory): add_bathymetry=False, allow_time_extrapolation=False, verbose_progress=True, + buffer_spec=buffer_spec, + limit_spec=None, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 311d83f61..c290f2775 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -25,9 +25,7 @@ class Instrument(abc.ABC): """Base class for instruments and their simulation.""" #! TODO List: - # TODO: update documentation/quickstart - # TODO: update tests - #! TODO: how is this handling credentials?! Seems to work already, are these set up from my previous instances of using copernicusmarine? Therefore users will only have to do it once too? + # TODO: how is this handling credentials?! Seems to work already, are these set up from my previous instances of using copernicusmarine? Therefore users will only have to do it once too? def __init__( self, @@ -39,7 +37,8 @@ def __init__( add_bathymetry: bool, allow_time_extrapolation: bool, verbose_progress: bool, - bathymetry_file: str = "bathymetry.nc", + buffer_spec: dict | None = None, + limit_spec: dict | None = None, ): """Initialise instrument.""" self.name = name @@ -53,10 +52,11 @@ def __init__( "time": "time", "depth": "depth", } # same dimensions for all instruments - self.bathymetry_file = bathymetry_file self.add_bathymetry = add_bathymetry self.allow_time_extrapolation = allow_time_extrapolation self.verbose_progress = verbose_progress + self.buffer_spec = buffer_spec + self.limit_spec = limit_spec def load_input_data(self) -> FieldSet: """Load and return the input data as a FieldSet for the instrument.""" @@ -64,20 +64,11 @@ def load_input_data(self) -> FieldSet: datasets = [] for var in self.variables.values(): physical = True if var in COPERNICUSMARINE_PHYS_VARIABLES else False - - # TODO: TEMPORARY BODGE FOR DRIFTER INSTRUMENT - REMOVE WHEN ABLE TO! - if self.name == "Drifter": - ds = self._get_copernicus_ds_DRIFTER(physical=physical, var=var) - else: - ds = self._get_copernicus_ds(physical=physical, var=var) - datasets.append(ds) + datasets.append(self._get_copernicus_ds(physical=physical, var=var)) # make sure time dims are matched if BGC variables are present (different monthly/daily resolutions can impact fieldset_endtime in simulate) - if any( - key in COPERNICUSMARINE_BGC_VARIABLES - for key in ds.keys() - for ds in datasets - ): + all_keys = set().union(*(ds.keys() for ds in datasets)) + if all_keys & set(COPERNICUSMARINE_BGC_VARIABLES): datasets = self._align_temporal(datasets) ds_concat = xr.merge(datasets) # TODO: deal with WARNINGS? @@ -100,11 +91,15 @@ def load_input_data(self) -> FieldSet: g.negate_depth() # bathymetry data - bathymetry_field = _get_bathy_data( - self.expedition.schedule.space_time_region - ).bathymetry - bathymetry_field.data = -bathymetry_field.data - fieldset.add_field(bathymetry_field) + if self.add_bathymetry: + bathymetry_field = _get_bathy_data( + self.expedition.schedule.space_time_region, + latlon_buffer=self.buffer_spec.get("latlon") + if self.buffer_spec + else None, + ).bathymetry + bathymetry_field.data = -bathymetry_field.data + fieldset.add_field(bathymetry_field) return fieldset @@ -127,8 +122,6 @@ def execute(self, measurements: list, out_path: str | Path) -> None: self.simulate(measurements, out_path) print("\n") - # self.simulate(measurements, out_path) - def _get_copernicus_ds( self, physical: bool, @@ -142,50 +135,28 @@ def _get_copernicus_ds( variable=var if not physical else None, ) - return copernicusmarine.open_dataset( - dataset_id=product_id, - dataset_part="default", - minimum_longitude=self.expedition.schedule.space_time_region.spatial_range.minimum_longitude, - maximum_longitude=self.expedition.schedule.space_time_region.spatial_range.maximum_longitude, - minimum_latitude=self.expedition.schedule.space_time_region.spatial_range.minimum_latitude, - maximum_latitude=self.expedition.schedule.space_time_region.spatial_range.maximum_latitude, - variables=[var], - start_datetime=self.expedition.schedule.space_time_region.time_range.start_time, - end_datetime=self.expedition.schedule.space_time_region.time_range.end_time, - coordinates_selection_method="outside", - ) + latlon_buffer = self.buffer_spec.get("latlon") if self.buffer_spec else 0.0 + time_buffer = self.buffer_spec.get("time") if self.buffer_spec else 0.0 - # TODO: TEMPORARY BODGE FOR DRIFTER INSTRUMENT - REMOVE WHEN ABLE TO! - def _get_copernicus_ds_DRIFTER( - self, - physical: bool, - var: str, - ) -> xr.Dataset: - """Get Copernicus Marine dataset for direct ingestion.""" - product_id = _select_product_id( - physical=physical, - schedule_start=self.expedition.schedule.space_time_region.time_range.start_time, - schedule_end=self.expedition.schedule.space_time_region.time_range.end_time, - variable=var if not physical else None, - ) + depth_min = self.limit_spec.get("depth_min") if self.limit_spec else None + depth_max = self.limit_spec.get("depth_max") if self.limit_spec else None return copernicusmarine.open_dataset( dataset_id=product_id, - dataset_part="default", minimum_longitude=self.expedition.schedule.space_time_region.spatial_range.minimum_longitude - - 3.0, + - latlon_buffer, maximum_longitude=self.expedition.schedule.space_time_region.spatial_range.maximum_longitude - + 3.0, + + latlon_buffer, minimum_latitude=self.expedition.schedule.space_time_region.spatial_range.minimum_latitude - - 3.0, + - latlon_buffer, maximum_latitude=self.expedition.schedule.space_time_region.spatial_range.maximum_latitude - + 3.0, - maximum_depth=1.0, - minimum_depth=1.0, + + latlon_buffer, variables=[var], start_datetime=self.expedition.schedule.space_time_region.time_range.start_time, end_datetime=self.expedition.schedule.space_time_region.time_range.end_time - + timedelta(days=21.0), + + timedelta(days=time_buffer), + minimum_depth=depth_min, + maximum_depth=depth_max, coordinates_selection_method="outside", ) diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index d205795b0..9bd923539 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -96,6 +96,8 @@ def __init__(self, expedition, directory): add_bathymetry=True, allow_time_extrapolation=True, verbose_progress=False, + buffer_spec=None, + limit_spec=None, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 182d274c4..4c286f3c7 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -130,6 +130,8 @@ def __init__(self, expedition, directory): add_bathymetry=True, allow_time_extrapolation=True, verbose_progress=False, + buffer_spec=None, + limit_spec=None, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index 06f6b71f7..fee4e3264 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -71,6 +71,15 @@ def __init__(self, expedition, directory): "T": f"{Drifter.name}_t.nc", } variables = {"U": "uo", "V": "vo", "T": "thetao"} + buffer_spec = { + "latlon": 6.0, # [degrees] + "time": 21.0, # [days] + } + limit_spec = { + "depth_min": 1.0, # [meters] + "depth_max": 1.0, # [meters] + } + super().__init__( Drifter.name, expedition, @@ -80,6 +89,8 @@ def __init__(self, expedition, directory): add_bathymetry=False, allow_time_extrapolation=False, verbose_progress=True, + buffer_spec=buffer_spec, + limit_spec=limit_spec, ) def simulate(self, measurements, out_path) -> None: @@ -121,7 +132,7 @@ def simulate(self, measurements, out_path) -> None: chunks=[len(drifter_particleset), 100], ) - # get earliest between fieldset end time and provide end time + # get earliest between fieldset end time and prescribed end time fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) if ENDTIME is None: actual_endtime = fieldset_endtime diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index f6099869d..32dcdd4f3 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -72,6 +72,8 @@ def __init__(self, expedition, directory): add_bathymetry=False, allow_time_extrapolation=True, verbose_progress=False, + buffer_spec=None, + limit_spec=None, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index fd88240d4..2d6a70798 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -95,6 +95,8 @@ def __init__(self, expedition, directory): add_bathymetry=True, allow_time_extrapolation=True, verbose_progress=False, + buffer_spec=None, + limit_spec=None, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index aa310a97c..0cb139559 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -132,7 +132,7 @@ def verify( if not ignore_missing_bathymetry: try: bathymetry_field = _get_bathy_data( - self.space_time_region + self.space_time_region, latlon_buffer=None ).bathymetry # via copernicusmarine except Exception as e: raise ScheduleError( @@ -406,9 +406,6 @@ def verify(self, expedition: Expedition) -> None: hasattr(self, config_attr) and inst_type not in instruments_in_expedition ): - print( - f"{inst_type.value} configuration provided but not in schedule. Removing config." - ) setattr(self, config_attr, None) # Check all scheduled instruments are configured for inst_type in instruments_in_expedition: diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index 4ea17c4a5..7e8630348 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -412,14 +412,18 @@ def _start_end_in_product_timerange( ) -def _get_bathy_data(space_time_region) -> FieldSet: +def _get_bathy_data(space_time_region, latlon_buffer: float | None = None) -> FieldSet: """Bathymetry data 'streamed' directly from Copernicus Marine.""" ds_bathymetry = copernicusmarine.open_dataset( dataset_id="cmems_mod_glo_phy_my_0.083deg_static", - minimum_longitude=space_time_region.spatial_range.minimum_longitude, - maximum_longitude=space_time_region.spatial_range.maximum_longitude, - minimum_latitude=space_time_region.spatial_range.minimum_latitude, - maximum_latitude=space_time_region.spatial_range.maximum_latitude, + minimum_longitude=space_time_region.spatial_range.minimum_longitude + - (latlon_buffer if latlon_buffer is not None else 0), + maximum_longitude=space_time_region.spatial_range.maximum_longitude + + (latlon_buffer if latlon_buffer is not None else 0), + minimum_latitude=space_time_region.spatial_range.minimum_latitude + - (latlon_buffer if latlon_buffer is not None else 0), + maximum_latitude=space_time_region.spatial_range.maximum_latitude + + (latlon_buffer if latlon_buffer is not None else 0), variables=["deptho"], start_datetime=space_time_region.time_range.start_time, end_datetime=space_time_region.time_range.end_time, From 43afe97e19e9162c02da8225250b37a9c349559e Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Fri, 7 Nov 2025 13:18:24 +0100 Subject: [PATCH 57/97] remove _creds.py --- src/virtualship/cli/_creds.py | 108 ---------------------------------- 1 file changed, 108 deletions(-) delete mode 100644 src/virtualship/cli/_creds.py diff --git a/src/virtualship/cli/_creds.py b/src/virtualship/cli/_creds.py deleted file mode 100644 index d9169ec4e..000000000 --- a/src/virtualship/cli/_creds.py +++ /dev/null @@ -1,108 +0,0 @@ -# - - -# TODO: TO DELETE?! - -from __future__ import annotations - -from pathlib import Path - -import click -import pydantic -import yaml - -from virtualship.errors import CredentialFileError - -CREDENTIALS_FILE = "credentials.yaml" - - -class Credentials(pydantic.BaseModel): - """Credentials to be used in `virtualship fetch` command.""" - - COPERNICUS_USERNAME: str - COPERNICUS_PASSWORD: str - - @classmethod - def from_yaml(cls, path: str | Path) -> Credentials: - """ - Load credentials from a yaml file. - - :param path: Path to the file to load from. - :returns Credentials: The credentials. - """ - with open(path) as file: - data = yaml.safe_load(file) - - if not isinstance(data, dict): - raise CredentialFileError("Credential file is of an invalid format.") - - return cls(**data) - - def dump(self) -> str: - """ - Dump credentials to a yaml string. - - :param creds: The credentials to dump. - :returns str: The yaml string. - """ - return yaml.safe_dump(self.model_dump()) - - def to_yaml(self, path: str | Path) -> None: - """ - Write credentials to a yaml file. - - :param path: Path to the file to write to. - """ - with open(path, "w") as file: - file.write(self.dump()) - - -def get_dummy_credentials_yaml() -> str: - return ( - Credentials( - COPERNICUS_USERNAME="my_username", COPERNICUS_PASSWORD="my_password" - ) - .dump() - .strip() - ) - - -def get_credentials_flow( - username: str | None, password: str | None, creds_path: Path -) -> tuple[str, str]: - """ - Execute flow of getting credentials for use in the `fetch` command. - - - If username and password are provided via CLI, use them (ignore the credentials file if exists). - - If username and password are not provided, try to load them from the credentials file. - - If no credentials are provided, print a message on how to make credentials file and prompt for credentials. - - :param username: The username provided via CLI. - :param password: The password provided via CLI. - :param creds_path: The path to the credentials file. - """ - if username and password: - if creds_path.exists(): - click.echo( - f"Credentials file exists at '{creds_path}', but username and password are already provided. Ignoring credentials file." - ) - return username, password - - try: - creds = Credentials.from_yaml(creds_path) - click.echo(f"Loaded credentials from '{creds_path}'.") - return creds.COPERNICUS_USERNAME, creds.COPERNICUS_PASSWORD - except FileNotFoundError: - msg = f"""Credentials not provided. Credentials can be obtained from https://data.marine.copernicus.eu/register. Either pass in via `--username` and `--password` arguments, or via config file at '{creds_path}'. Config file should be YAML along following format: -### {creds_path} - -{get_dummy_credentials_yaml().strip()} - -### - -Prompting for credentials instead... -""" - click.echo(msg) - username = click.prompt("username") - password = click.prompt("password", hide_input=True) - return username, password From e9a0c54e16525f4533aee918c189c37525763eca Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Fri, 7 Nov 2025 15:13:23 +0100 Subject: [PATCH 58/97] CTD_BGC fieldset bug fix --- src/virtualship/cli/_plan.py | 1 - src/virtualship/instruments/base.py | 83 ++++++++++++-------------- src/virtualship/instruments/ctd_bgc.py | 2 - 3 files changed, 38 insertions(+), 48 deletions(-) diff --git a/src/virtualship/cli/_plan.py b/src/virtualship/cli/_plan.py index 6aa6ff28e..5ae46a82f 100644 --- a/src/virtualship/cli/_plan.py +++ b/src/virtualship/cli/_plan.py @@ -1046,7 +1046,6 @@ def save_pressed(self) -> None: # verify schedule expedition_editor.expedition.schedule.verify( ship_speed_value, - bathy_data_dir=None, check_space_time_region=True, ignore_missing_bathymetry=True, ) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index c290f2775..e597174b2 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -4,13 +4,11 @@ from typing import TYPE_CHECKING import copernicusmarine -import numpy as np import xarray as xr from yaspin import yaspin from parcels import FieldSet from virtualship.utils import ( - COPERNICUSMARINE_BGC_VARIABLES, COPERNICUSMARINE_PHYS_VARIABLES, _get_bathy_data, _select_product_id, @@ -61,31 +59,16 @@ def __init__( def load_input_data(self) -> FieldSet: """Load and return the input data as a FieldSet for the instrument.""" try: - datasets = [] - for var in self.variables.values(): - physical = True if var in COPERNICUSMARINE_PHYS_VARIABLES else False - datasets.append(self._get_copernicus_ds(physical=physical, var=var)) - - # make sure time dims are matched if BGC variables are present (different monthly/daily resolutions can impact fieldset_endtime in simulate) - all_keys = set().union(*(ds.keys() for ds in datasets)) - if all_keys & set(COPERNICUSMARINE_BGC_VARIABLES): - datasets = self._align_temporal(datasets) - - ds_concat = xr.merge(datasets) # TODO: deal with WARNINGS? - - fieldset = FieldSet.from_xarray_dataset( - ds_concat, self.variables, self.dimensions, mesh="spherical" - ) - + fieldset = self._generate_fieldset() except Exception as e: raise FileNotFoundError( - f"Failed to load input data directly from Copernicus Marine for instrument '{self.name}'. " - f"Please check your credentials, network connection, and variable names. Original error: {e}" + f"Failed to load input data directly from Copernicus Marine for instrument '{self.name}'.Original error: {e}" ) from e # interpolation methods for var in (v for v in self.variables if v not in ("U", "V")): getattr(fieldset, var).interp_method = "linear_invdist_land_tracer" + # depth negative for g in fieldset.gridset.grids: g.negate_depth() @@ -109,18 +92,22 @@ def simulate(self, data_dir: Path, measurements: list, out_path: str | Path): def execute(self, measurements: list, out_path: str | Path) -> None: """Run instrument simulation.""" - if not self.verbose_progress: - with yaspin( - text=f"Simulating {self.name} measurements... ", - side="right", - spinner=ship_spinner, - ) as spinner: + TMP = False + if not TMP: + if not self.verbose_progress: + with yaspin( + text=f"Simulating {self.name} measurements... ", + side="right", + spinner=ship_spinner, + ) as spinner: + self.simulate(measurements, out_path) + spinner.ok("✅\n") + else: + print(f"Simulating {self.name} measurements... ") self.simulate(measurements, out_path) - spinner.ok("✅\n") + print("\n") else: - print(f"Simulating {self.name} measurements... ") self.simulate(measurements, out_path) - print("\n") def _get_copernicus_ds( self, @@ -160,19 +147,25 @@ def _get_copernicus_ds( coordinates_selection_method="outside", ) - def _align_temporal(self, datasets: list[xr.Dataset]) -> list[xr.Dataset]: - """Align monthly and daily time dims of multiple datasets (by repeating monthly values daily).""" - reference_time = datasets[ - np.argmax(ds.time for ds in datasets) - ].time # daily timeseries - - datasets_aligned = [] - for ds in datasets: - if not np.array_equal(ds.time, reference_time): - # TODO: NEED TO CHOOSE BEST METHOD HERE - # ds = ds.resample(time="1D").ffill().reindex(time=reference_time) - # ds = ds.resample(time="1D").ffill() - ds = ds.reindex({"time": reference_time}, method="nearest") - datasets_aligned.append(ds) - - return datasets_aligned + def _generate_fieldset(self) -> FieldSet: + """ + Fieldset per variable then combine. + + Avoids issues when creating one FieldSet of ds's sourced from different Copernicus Marine product IDs, which is often the case for BGC variables. + + """ + fieldsets_list = [] + for key, var in self.variables.items(): + physical = True if var in COPERNICUSMARINE_PHYS_VARIABLES else False + ds = self._get_copernicus_ds(physical=physical, var=var) + fieldset = FieldSet.from_xarray_dataset( + ds, {key: var}, self.dimensions, mesh="spherical" + ) + fieldsets_list.append(fieldset) + base_fieldset = fieldsets_list[0] + if len(fieldsets_list) > 1: + for fs, key in zip( + fieldsets_list[1:], list(self.variables.keys())[1:], strict=True + ): + base_fieldset.add_field(getattr(fs, key)) + return base_fieldset diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 4c286f3c7..33c17d99a 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -205,8 +205,6 @@ def simulate(self, measurements, out_path) -> None: # define output file for the simulation out_file = ctd_bgc_particleset.ParticleFile(name=out_path, outputdt=OUTPUT_DT) - breakpoint() - # execute simulation ctd_bgc_particleset.execute( [ From 995a6e57635570fb1b1faf8cb59ca0c2a4360fb9 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 10 Nov 2025 09:52:40 +0100 Subject: [PATCH 59/97] fixing bugs associated with BGC data access --- src/virtualship/cli/_run.py | 11 ++-- src/virtualship/instruments/argo_float.py | 2 +- src/virtualship/instruments/base.py | 62 ++++++++++++----------- 3 files changed, 38 insertions(+), 37 deletions(-) diff --git a/src/virtualship/cli/_run.py b/src/virtualship/cli/_run.py index df8a24fdb..29bf0f7bb 100644 --- a/src/virtualship/cli/_run.py +++ b/src/virtualship/cli/_run.py @@ -3,6 +3,7 @@ import logging import os import shutil +import time from pathlib import Path import pyproj @@ -39,12 +40,9 @@ def _run(expedition_dir: str | Path) -> None: :param expedition_dir: The base directory for the expedition. """ - # ################################# TEMPORARY TIMER: START ################################# - import time - + # start timing start_time = time.time() print("[TIMER] Expedition started...") - # ################################# TEMPORARY TIMER: START ################################# print("\n╔═════════════════════════════════════════════════╗") print("║ VIRTUALSHIP EXPEDITION STATUS ║") @@ -139,11 +137,10 @@ def _run(expedition_dir: str | Path) -> None: ) print("\n------------- END -------------\n") - ################################# TEMPORARY TIMER: END ################################# + # end timing end_time = time.time() elapsed = end_time - start_time - print(f"[TIMER] Expedition completed in {elapsed:.2f} seconds.") - ################################# TEMPORARY TIMER: END ################################# + print(f"[TIMER] Expedition completed in {elapsed / 60.0:.2f} minutes.") def _load_checkpoint(expedition_dir: Path) -> Checkpoint | None: diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index 12ee8945a..7935ca550 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -147,7 +147,7 @@ def __init__(self, expedition, directory): } variables = {"U": "uo", "V": "vo", "S": "so", "T": "thetao"} buffer_spec = { - "latlon": 6.0, # [degrees] + "latlon": 3.0, # [degrees] "time": 21.0, # [days] } diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index e597174b2..06d8b8c02 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -1,4 +1,5 @@ import abc +from collections import OrderedDict from datetime import timedelta from pathlib import Path from typing import TYPE_CHECKING @@ -8,6 +9,7 @@ from yaspin import yaspin from parcels import FieldSet +from virtualship.errors import CopernicusCatalogueError from virtualship.utils import ( COPERNICUSMARINE_PHYS_VARIABLES, _get_bathy_data, @@ -43,7 +45,8 @@ def __init__( self.expedition = expedition self.directory = directory self.filenames = filenames - self.variables = variables + + self.variables = OrderedDict(variables) self.dimensions = { "lon": "longitude", "lat": "latitude", @@ -61,8 +64,8 @@ def load_input_data(self) -> FieldSet: try: fieldset = self._generate_fieldset() except Exception as e: - raise FileNotFoundError( - f"Failed to load input data directly from Copernicus Marine for instrument '{self.name}'.Original error: {e}" + raise CopernicusCatalogueError( + f"Failed to load input data directly from Copernicus Marine for instrument '{self.name}'. Original error: {e}" ) from e # interpolation methods @@ -92,22 +95,18 @@ def simulate(self, data_dir: Path, measurements: list, out_path: str | Path): def execute(self, measurements: list, out_path: str | Path) -> None: """Run instrument simulation.""" - TMP = False - if not TMP: - if not self.verbose_progress: - with yaspin( - text=f"Simulating {self.name} measurements... ", - side="right", - spinner=ship_spinner, - ) as spinner: - self.simulate(measurements, out_path) - spinner.ok("✅\n") - else: - print(f"Simulating {self.name} measurements... ") + if not self.verbose_progress: + with yaspin( + text=f"Simulating {self.name} measurements... ", + side="right", + spinner=ship_spinner, + ) as spinner: self.simulate(measurements, out_path) - print("\n") + spinner.ok("✅\n") else: + print(f"Simulating {self.name} measurements... ") self.simulate(measurements, out_path) + print("\n") def _get_copernicus_ds( self, @@ -122,11 +121,10 @@ def _get_copernicus_ds( variable=var if not physical else None, ) - latlon_buffer = self.buffer_spec.get("latlon") if self.buffer_spec else 0.0 - time_buffer = self.buffer_spec.get("time") if self.buffer_spec else 0.0 - - depth_min = self.limit_spec.get("depth_min") if self.limit_spec else None - depth_max = self.limit_spec.get("depth_max") if self.limit_spec else None + latlon_buffer = self._get_spec_value("buffer", "latlon", 0.0) + time_buffer = self._get_spec_value("buffer", "time", 0.0) + depth_min = self._get_spec_value("limit", "depth_min", None) + depth_max = self._get_spec_value("limit", "depth_max", None) return copernicusmarine.open_dataset( dataset_id=product_id, @@ -151,21 +149,27 @@ def _generate_fieldset(self) -> FieldSet: """ Fieldset per variable then combine. - Avoids issues when creating one FieldSet of ds's sourced from different Copernicus Marine product IDs, which is often the case for BGC variables. - + Avoids issues when creating directly one FieldSet of ds's sourced from different Copernicus Marine product IDs, which is often the case for BGC variables. """ fieldsets_list = [] - for key, var in self.variables.items(): + keys = list(self.variables.keys()) + for key in keys: + var = self.variables[key] physical = True if var in COPERNICUSMARINE_PHYS_VARIABLES else False ds = self._get_copernicus_ds(physical=physical, var=var) - fieldset = FieldSet.from_xarray_dataset( + fs = FieldSet.from_xarray_dataset( ds, {key: var}, self.dimensions, mesh="spherical" ) - fieldsets_list.append(fieldset) + fieldsets_list.append(fs) + base_fieldset = fieldsets_list[0] if len(fieldsets_list) > 1: - for fs, key in zip( - fieldsets_list[1:], list(self.variables.keys())[1:], strict=True - ): + for fs, key in zip(fieldsets_list[1:], keys[1:], strict=True): base_fieldset.add_field(getattr(fs, key)) + return base_fieldset + + def _get_spec_value(self, spec_type: str, key: str, default=None): + """Helper to extract a value from buffer_spec or limit_spec.""" + spec = self.buffer_spec if spec_type == "buffer" else self.limit_spec + return spec.get(key) if spec and spec.get(key) is not None else default From ec3329672ddcf9fad9d33fa21ab714651024aa54 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 10 Nov 2025 10:18:41 +0100 Subject: [PATCH 60/97] update dependencies --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 9862463b5..7f9a2108a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,7 @@ dependencies = [ "copernicusmarine >= 2.2.2", "yaspin", "textual", + "openpyxl", ] [project.urls] From c1321ba25d5507ffc2ee75d49ede93b51e2be5d6 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 10 Nov 2025 10:32:53 +0100 Subject: [PATCH 61/97] logic for handling copernicus credentials --- src/virtualship/cli/_run.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/virtualship/cli/_run.py b/src/virtualship/cli/_run.py index 29bf0f7bb..bf98f562a 100644 --- a/src/virtualship/cli/_run.py +++ b/src/virtualship/cli/_run.py @@ -6,6 +6,7 @@ import time from pathlib import Path +import copernicusmarine import pyproj from virtualship.expedition.simulate_schedule import ( @@ -48,6 +49,21 @@ def _run(expedition_dir: str | Path) -> None: print("║ VIRTUALSHIP EXPEDITION STATUS ║") print("╚═════════════════════════════════════════════════╝") + COPERNICUS_CREDS_FILE = os.path.expandvars( + "$HOME/.copernicusmarine/.copernicusmarine-credentials" + ) + + if ( + os.path.isfile(COPERNICUS_CREDS_FILE) + and os.path.getsize(COPERNICUS_CREDS_FILE) > 0 + ): + pass + else: + print( + "\nPlease enter your log in details for the Copernicus Marine Service (only necessary the first time you run VirtualShip). \n\nIf you have not registered yet, please do so at https://marine.copernicus.eu/.\n" + ) + copernicusmarine.login() + if isinstance(expedition_dir, str): expedition_dir = Path(expedition_dir) From f386f42aad476950f8f0c863c202d359dc4be573 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 10 Nov 2025 15:10:20 +0100 Subject: [PATCH 62/97] add support for taking local pre-downloaded data with --from-data optional flag in virtualship run --- src/virtualship/cli/_plan.py | 2 +- src/virtualship/cli/_run.py | 30 ++--- src/virtualship/cli/commands.py | 15 ++- src/virtualship/instruments/adcp.py | 3 +- src/virtualship/instruments/argo_float.py | 3 +- src/virtualship/instruments/base.py | 116 ++++++++++++++---- src/virtualship/instruments/ctd.py | 3 +- src/virtualship/instruments/ctd_bgc.py | 3 +- src/virtualship/instruments/drifter.py | 3 +- .../instruments/ship_underwater_st.py | 3 +- src/virtualship/instruments/xbt.py | 3 +- src/virtualship/models/expedition.py | 9 +- src/virtualship/utils.py | 66 +++++++--- 13 files changed, 190 insertions(+), 69 deletions(-) diff --git a/src/virtualship/cli/_plan.py b/src/virtualship/cli/_plan.py index 5ae46a82f..a164cba31 100644 --- a/src/virtualship/cli/_plan.py +++ b/src/virtualship/cli/_plan.py @@ -1047,7 +1047,7 @@ def save_pressed(self) -> None: expedition_editor.expedition.schedule.verify( ship_speed_value, check_space_time_region=True, - ignore_missing_bathymetry=True, + ignore_land_test=True, ) expedition_saved = expedition_editor.save_changes() diff --git a/src/virtualship/cli/_run.py b/src/virtualship/cli/_run.py index bf98f562a..fac091421 100644 --- a/src/virtualship/cli/_run.py +++ b/src/virtualship/cli/_run.py @@ -35,7 +35,7 @@ logging.getLogger("copernicusmarine").setLevel("ERROR") -def _run(expedition_dir: str | Path) -> None: +def _run(expedition_dir: str | Path, from_data: Path | None = None) -> None: """ Perform an expedition, providing terminal feedback and file output. @@ -49,20 +49,21 @@ def _run(expedition_dir: str | Path) -> None: print("║ VIRTUALSHIP EXPEDITION STATUS ║") print("╚═════════════════════════════════════════════════╝") - COPERNICUS_CREDS_FILE = os.path.expandvars( - "$HOME/.copernicusmarine/.copernicusmarine-credentials" - ) - - if ( - os.path.isfile(COPERNICUS_CREDS_FILE) - and os.path.getsize(COPERNICUS_CREDS_FILE) > 0 - ): - pass - else: - print( - "\nPlease enter your log in details for the Copernicus Marine Service (only necessary the first time you run VirtualShip). \n\nIf you have not registered yet, please do so at https://marine.copernicus.eu/.\n" + if from_data is None: + COPERNICUS_CREDS_FILE = os.path.expandvars( + "$HOME/.copernicusmarine/.copernicusmarine-credentials" ) - copernicusmarine.login() + + if ( + os.path.isfile(COPERNICUS_CREDS_FILE) + and os.path.getsize(COPERNICUS_CREDS_FILE) > 0 + ): + pass + else: + print( + "\nPlease enter your log in details for the Copernicus Marine Service (only necessary the first time you run VirtualShip). \n\nIf you have not registered yet, please do so at https://marine.copernicus.eu/.\n" + ) + copernicusmarine.login() if isinstance(expedition_dir, str): expedition_dir = Path(expedition_dir) @@ -136,6 +137,7 @@ def _run(expedition_dir: str | Path) -> None: instrument = instrument_class( expedition=expedition, directory=expedition_dir, + from_data=Path(from_data) if from_data is not None else None, ) # execute simulation diff --git a/src/virtualship/cli/commands.py b/src/virtualship/cli/commands.py index 2b0ce8e93..f2f9f272d 100644 --- a/src/virtualship/cli/commands.py +++ b/src/virtualship/cli/commands.py @@ -6,6 +6,8 @@ from virtualship.cli._plan import _plan from virtualship.cli._run import _run from virtualship.utils import ( + COPERNICUSMARINE_BGC_VARIABLES, + COPERNICUSMARINE_PHYS_VARIABLES, EXPEDITION, mfp_to_yaml, ) @@ -81,6 +83,15 @@ def plan(path): "path", type=click.Path(exists=True, file_okay=False, dir_okay=True, readable=True), ) -def run(path): +@click.option( + "--from-data", + type=str, + default=None, + help="Use pre-downloaded data, saved to disk, for expedition, instead of streaming directly via Copernicus Marine" + "Assumes all data is stored in prescribed directory, and all variables (as listed below) are present." + f"Required variables are: {set(COPERNICUSMARINE_PHYS_VARIABLES + COPERNICUSMARINE_BGC_VARIABLES)}" + "Assumes that variable names at least contain the standard Copernicus Marine variable name as a substring.", +) +def run(path, from_data): """Execute the expedition simulations.""" - _run(Path(path)) + _run(Path(path), from_data) diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index dd675b7ee..8dca31a3e 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -54,7 +54,7 @@ def _sample_velocity(particle, fieldset, time): class ADCPInstrument(Instrument): """ADCP instrument class.""" - def __init__(self, expedition, directory): + def __init__(self, expedition, directory, from_data): """Initialize ADCPInstrument.""" filenames = { "U": f"{ADCP.name}_uv.nc", @@ -73,6 +73,7 @@ def __init__(self, expedition, directory): verbose_progress=False, buffer_spec=None, limit_spec=None, + from_data=from_data, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index 7935ca550..8ba794ba2 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -137,7 +137,7 @@ def _check_error(particle, fieldset, time): class ArgoFloatInstrument(Instrument): """ArgoFloat instrument class.""" - def __init__(self, expedition, directory): + def __init__(self, expedition, directory, from_data): """Initialize ArgoFloatInstrument.""" filenames = { "U": f"{ArgoFloat.name}_uv.nc", @@ -162,6 +162,7 @@ def __init__(self, expedition, directory): verbose_progress=True, buffer_spec=buffer_spec, limit_spec=None, + from_data=from_data, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 06d8b8c02..7941bf186 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -12,6 +12,7 @@ from virtualship.errors import CopernicusCatalogueError from virtualship.utils import ( COPERNICUSMARINE_PHYS_VARIABLES, + _find_nc_file_with_variable, _get_bathy_data, _select_product_id, ship_spinner, @@ -24,9 +25,6 @@ class Instrument(abc.ABC): """Base class for instruments and their simulation.""" - #! TODO List: - # TODO: how is this handling credentials?! Seems to work already, are these set up from my previous instances of using copernicusmarine? Therefore users will only have to do it once too? - def __init__( self, name: str, @@ -37,6 +35,7 @@ def __init__( add_bathymetry: bool, allow_time_extrapolation: bool, verbose_progress: bool, + from_data: Path | None, buffer_spec: dict | None = None, limit_spec: dict | None = None, ): @@ -45,6 +44,7 @@ def __init__( self.expedition = expedition self.directory = directory self.filenames = filenames + self.from_data = from_data self.variables = OrderedDict(variables) self.dimensions = { @@ -65,7 +65,7 @@ def load_input_data(self) -> FieldSet: fieldset = self._generate_fieldset() except Exception as e: raise CopernicusCatalogueError( - f"Failed to load input data directly from Copernicus Marine for instrument '{self.name}'. Original error: {e}" + f"Failed to load input data directly from Copernicus Marine (or local data) for instrument '{self.name}'. Original error: {e}" ) from e # interpolation methods @@ -83,6 +83,7 @@ def load_input_data(self) -> FieldSet: latlon_buffer=self.buffer_spec.get("latlon") if self.buffer_spec else None, + from_data=self.from_data, ).bathymetry bathymetry_field.data = -bathymetry_field.data fieldset.add_field(bathymetry_field) @@ -90,23 +91,36 @@ def load_input_data(self) -> FieldSet: return fieldset @abc.abstractmethod - def simulate(self, data_dir: Path, measurements: list, out_path: str | Path): + def simulate( + self, + data_dir: Path, + measurements: list, + out_path: str | Path, + ) -> None: """Simulate instrument measurements.""" def execute(self, measurements: list, out_path: str | Path) -> None: """Run instrument simulation.""" - if not self.verbose_progress: - with yaspin( - text=f"Simulating {self.name} measurements... ", - side="right", - spinner=ship_spinner, - ) as spinner: + # + TMP = True # temporary spinner implementation + # + if TMP: + if not self.verbose_progress: + with yaspin( + text=f"Simulating {self.name} measurements... ", + side="right", + spinner=ship_spinner, + ) as spinner: + self.simulate(measurements, out_path) + spinner.ok("✅\n") + else: + print(f"Simulating {self.name} measurements... ") self.simulate(measurements, out_path) - spinner.ok("✅\n") + print("\n") + + # else: - print(f"Simulating {self.name} measurements... ") self.simulate(measurements, out_path) - print("\n") def _get_copernicus_ds( self, @@ -145,27 +159,81 @@ def _get_copernicus_ds( coordinates_selection_method="outside", ) + def _load_local_ds(self, filename) -> xr.Dataset: + """ + Load local dataset from specified data directory. + + Sliced according to expedition.schedule.space_time_region andbuffer and limit specs. + """ + ds = xr.open_dataset(self.from_data.joinpath(filename)) + + coord_rename = {} + if "lat" in ds.coords: + coord_rename["lat"] = "latitude" + if "lon" in ds.coords: + coord_rename["lon"] = "longitude" + if coord_rename: + ds = ds.rename(coord_rename) + + min_lon = ( + self.expedition.schedule.space_time_region.spatial_range.minimum_longitude + - self._get_spec_value( + "buffer", "latlon", 3.0 + ) # always add min 3 deg buffer for local data to avoid edge issues with ds.sel() + ) + max_lon = ( + self.expedition.schedule.space_time_region.spatial_range.maximum_longitude + + self._get_spec_value("buffer", "latlon", 3.0) + ) + min_lat = ( + self.expedition.schedule.space_time_region.spatial_range.minimum_latitude + - self._get_spec_value("buffer", "latlon", 3.0) + ) + max_lat = ( + self.expedition.schedule.space_time_region.spatial_range.maximum_latitude + + self._get_spec_value("buffer", "latlon", 3.0) + ) + min_depth = self._get_spec_value("limit", "depth_min", None) + max_depth = self._get_spec_value("limit", "depth_max", None) + + return ds.sel( + latitude=slice(min_lat, max_lat), + longitude=slice(min_lon, max_lon), + depth=slice(min_depth, max_depth) + if min_depth is not None and max_depth is not None + else slice(None), + ) + def _generate_fieldset(self) -> FieldSet: """ - Fieldset per variable then combine. + Create and combine FieldSets for each variable, supporting both local and Copernicus Marine data sources. - Avoids issues when creating directly one FieldSet of ds's sourced from different Copernicus Marine product IDs, which is often the case for BGC variables. + Avoids issues when using copernicusmarine and creating directly one FieldSet of ds's sourced from different Copernicus Marine product IDs, which is often the case for BGC variables. """ fieldsets_list = [] keys = list(self.variables.keys()) + for key in keys: var = self.variables[key] - physical = True if var in COPERNICUSMARINE_PHYS_VARIABLES else False - ds = self._get_copernicus_ds(physical=physical, var=var) - fs = FieldSet.from_xarray_dataset( - ds, {key: var}, self.dimensions, mesh="spherical" - ) + if self.from_data is not None: # load from local data + filename, full_var_name = _find_nc_file_with_variable( + self.from_data, var + ) + ds = self._load_local_ds(filename) + fs = FieldSet.from_xarray_dataset( + ds, {key: full_var_name}, self.dimensions, mesh="spherical" + ) + else: # steam via Copernicus Marine + physical = var in COPERNICUSMARINE_PHYS_VARIABLES + ds = self._get_copernicus_ds(physical=physical, var=var) + fs = FieldSet.from_xarray_dataset( + ds, {key: var}, self.dimensions, mesh="spherical" + ) fieldsets_list.append(fs) base_fieldset = fieldsets_list[0] - if len(fieldsets_list) > 1: - for fs, key in zip(fieldsets_list[1:], keys[1:], strict=True): - base_fieldset.add_field(getattr(fs, key)) + for fs, key in zip(fieldsets_list[1:], keys[1:], strict=False): + base_fieldset.add_field(getattr(fs, key)) return base_fieldset diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 9bd923539..04fb24edd 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -79,7 +79,7 @@ def _ctd_cast(particle, fieldset, time): class CTDInstrument(Instrument): """CTD instrument class.""" - def __init__(self, expedition, directory): + def __init__(self, expedition, directory, from_data): """Initialize CTDInstrument.""" filenames = { "S": f"{CTD.name}_s.nc", @@ -98,6 +98,7 @@ def __init__(self, expedition, directory): verbose_progress=False, buffer_spec=None, limit_spec=None, + from_data=from_data, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 33c17d99a..59dc4bc23 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -101,7 +101,7 @@ def _ctd_bgc_cast(particle, fieldset, time): class CTD_BGCInstrument(Instrument): """CTD_BGC instrument class.""" - def __init__(self, expedition, directory): + def __init__(self, expedition, directory, from_data): """Initialize CTD_BGCInstrument.""" filenames = { "o2": f"{CTD_BGC.name}_o2.nc", @@ -132,6 +132,7 @@ def __init__(self, expedition, directory): verbose_progress=False, buffer_spec=None, limit_spec=None, + from_data=from_data, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index fee4e3264..8ad0fe197 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -63,7 +63,7 @@ def _check_lifetime(particle, fieldset, time): class DrifterInstrument(Instrument): """Drifter instrument class.""" - def __init__(self, expedition, directory): + def __init__(self, expedition, directory, from_data): """Initialize DrifterInstrument.""" filenames = { "U": f"{Drifter.name}_uv.nc", @@ -91,6 +91,7 @@ def __init__(self, expedition, directory): verbose_progress=True, buffer_spec=buffer_spec, limit_spec=limit_spec, + from_data=from_data, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index 32dcdd4f3..a28ae12db 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -55,7 +55,7 @@ def _sample_temperature(particle, fieldset, time): class Underwater_STInstrument(Instrument): """Underwater_ST instrument class.""" - def __init__(self, expedition, directory): + def __init__(self, expedition, directory, from_data): """Initialize Underwater_STInstrument.""" filenames = { "S": f"{Underwater_ST.name}_s.nc", @@ -74,6 +74,7 @@ def __init__(self, expedition, directory): verbose_progress=False, buffer_spec=None, limit_spec=None, + from_data=from_data, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index 2d6a70798..5972efffe 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -77,7 +77,7 @@ def _xbt_cast(particle, fieldset, time): class XBTInstrument(Instrument): """XBT instrument class.""" - def __init__(self, expedition, directory): + def __init__(self, expedition, directory, from_data): """Initialize XBTInstrument.""" filenames = { "U": f"{XBT.name}_uv.nc", @@ -97,6 +97,7 @@ def __init__(self, expedition, directory): verbose_progress=False, buffer_spec=None, limit_spec=None, + from_data=from_data, ) def simulate(self, measurements, out_path) -> None: diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index 0cb139559..0b82e0af6 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -87,7 +87,7 @@ class Schedule(pydantic.BaseModel): def verify( self, ship_speed: float, - ignore_missing_bathymetry: bool = False, + ignore_land_test: bool = False, *, check_space_time_region: bool = False, ) -> None: @@ -129,11 +129,12 @@ def verify( # check if all waypoints are in water using bathymetry data # TODO: write test that checks that will flag when waypoint is on land!! [add to existing suite of fail .verify() tests in test_expedition.py] land_waypoints = [] - if not ignore_missing_bathymetry: + if not ignore_land_test: try: bathymetry_field = _get_bathy_data( - self.space_time_region, latlon_buffer=None - ).bathymetry # via copernicusmarine + self.space_time_region, + latlon_buffer=None, + ).bathymetry except Exception as e: raise ScheduleError( f"Problem loading bathymetry data (used to verify waypoints are in water) directly via copernicusmarine. \n\n original message: {e}" diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index 7e8630348..9f1a08e87 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -10,6 +10,7 @@ import copernicusmarine import numpy as np +import xarray as xr from parcels import FieldSet from virtualship.errors import CopernicusCatalogueError @@ -412,23 +413,41 @@ def _start_end_in_product_timerange( ) -def _get_bathy_data(space_time_region, latlon_buffer: float | None = None) -> FieldSet: - """Bathymetry data 'streamed' directly from Copernicus Marine.""" - ds_bathymetry = copernicusmarine.open_dataset( - dataset_id="cmems_mod_glo_phy_my_0.083deg_static", - minimum_longitude=space_time_region.spatial_range.minimum_longitude - - (latlon_buffer if latlon_buffer is not None else 0), - maximum_longitude=space_time_region.spatial_range.maximum_longitude - + (latlon_buffer if latlon_buffer is not None else 0), - minimum_latitude=space_time_region.spatial_range.minimum_latitude - - (latlon_buffer if latlon_buffer is not None else 0), - maximum_latitude=space_time_region.spatial_range.maximum_latitude - + (latlon_buffer if latlon_buffer is not None else 0), - variables=["deptho"], - start_datetime=space_time_region.time_range.start_time, - end_datetime=space_time_region.time_range.end_time, - coordinates_selection_method="outside", - ) +def _get_bathy_data( + space_time_region, latlon_buffer: float | None = None, from_data: Path | None = None +) -> FieldSet: + """Bathymetry data from local or 'streamed' directly from Copernicus Marine.""" + if from_data is not None: # load from local data + var = "deptho" + try: + filename, _ = _find_nc_file_with_variable(from_data, var) + except Exception as e: + raise RuntimeError( + f"Could not find bathymetry variable '{var}' in provided data directory '{from_data}'." + ) from e + ds_bathymetry = xr.open_dataset(from_data.joinpath(filename)) + bathymetry_variables = {"bathymetry": "deptho"} + bathymetry_dimensions = {"lon": "longitude", "lat": "latitude"} + return FieldSet.from_xarray_dataset( + ds_bathymetry, bathymetry_variables, bathymetry_dimensions + ) + + else: # stream via Copernicus Marine + ds_bathymetry = copernicusmarine.open_dataset( + dataset_id="cmems_mod_glo_phy_my_0.083deg_static", + minimum_longitude=space_time_region.spatial_range.minimum_longitude + - (latlon_buffer if latlon_buffer is not None else 0), + maximum_longitude=space_time_region.spatial_range.maximum_longitude + + (latlon_buffer if latlon_buffer is not None else 0), + minimum_latitude=space_time_region.spatial_range.minimum_latitude + - (latlon_buffer if latlon_buffer is not None else 0), + maximum_latitude=space_time_region.spatial_range.maximum_latitude + + (latlon_buffer if latlon_buffer is not None else 0), + variables=["deptho"], + start_datetime=space_time_region.time_range.start_time, + end_datetime=space_time_region.time_range.end_time, + coordinates_selection_method="outside", + ) bathymetry_variables = {"bathymetry": "deptho"} bathymetry_dimensions = {"lon": "longitude", "lat": "latitude"} return FieldSet.from_xarray_dataset( @@ -456,3 +475,16 @@ def expedition_cost(schedule_results: ScheduleOk, time_past: timedelta) -> float cost = ship_cost + argo_cost + drifter_cost return cost + + +def _find_nc_file_with_variable(data_dir: Path, var: str) -> str | None: + """Search for a .nc file in the given directory containing the specified variable.""" + for nc_file in data_dir.glob("*.nc"): + try: + with xr.open_dataset(nc_file, chunks={}) as ds: + matched_vars = [v for v in ds.variables if var in v] + if matched_vars: + return nc_file.name, matched_vars[0] + except Exception: + continue + return None From 7ba83fc30c23857361ca4a4a13f3ee70dcec914f Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 10 Nov 2025 15:45:09 +0100 Subject: [PATCH 63/97] update drifter to be at -1m depth, to avoid out of bounds at surface --- src/virtualship/instruments/base.py | 7 +------ src/virtualship/static/expedition.yaml | 2 +- tests/expedition/expedition_dir/expedition.yaml | 2 +- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 7941bf186..5c764cb3b 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -163,7 +163,7 @@ def _load_local_ds(self, filename) -> xr.Dataset: """ Load local dataset from specified data directory. - Sliced according to expedition.schedule.space_time_region andbuffer and limit specs. + Sliced according to expedition.schedule.space_time_region and buffer specs. """ ds = xr.open_dataset(self.from_data.joinpath(filename)) @@ -193,15 +193,10 @@ def _load_local_ds(self, filename) -> xr.Dataset: self.expedition.schedule.space_time_region.spatial_range.maximum_latitude + self._get_spec_value("buffer", "latlon", 3.0) ) - min_depth = self._get_spec_value("limit", "depth_min", None) - max_depth = self._get_spec_value("limit", "depth_max", None) return ds.sel( latitude=slice(min_lat, max_lat), longitude=slice(min_lon, max_lon), - depth=slice(min_depth, max_depth) - if min_depth is not None and max_depth is not None - else slice(None), ) def _generate_fieldset(self) -> FieldSet: diff --git a/src/virtualship/static/expedition.yaml b/src/virtualship/static/expedition.yaml index 1a9e39223..256bee87c 100644 --- a/src/virtualship/static/expedition.yaml +++ b/src/virtualship/static/expedition.yaml @@ -62,7 +62,7 @@ instruments_config: min_depth_meter: -11.0 stationkeeping_time_minutes: 20.0 drifter_config: - depth_meter: 0.0 + depth_meter: -1.0 lifetime_minutes: 60480.0 xbt_config: max_depth_meter: -285.0 diff --git a/tests/expedition/expedition_dir/expedition.yaml b/tests/expedition/expedition_dir/expedition.yaml index 9468028f8..0ed9e5f42 100644 --- a/tests/expedition/expedition_dir/expedition.yaml +++ b/tests/expedition/expedition_dir/expedition.yaml @@ -38,7 +38,7 @@ instruments_config: min_depth_meter: -11.0 stationkeeping_time_minutes: 20.0 drifter_config: - depth_meter: 0.0 + depth_meter: -1.0 lifetime_minutes: 40320.0 ship_underwater_st_config: period_minutes: 5.0 From f13c8363e5c2ac760716c02b7f919316d1d71c58 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 10 Nov 2025 15:45:19 +0100 Subject: [PATCH 64/97] tidy up --- src/virtualship/cli/_run.py | 2 ++ src/virtualship/cli/commands.py | 7 ++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/virtualship/cli/_run.py b/src/virtualship/cli/_run.py index fac091421..353f6a1fa 100644 --- a/src/virtualship/cli/_run.py +++ b/src/virtualship/cli/_run.py @@ -50,6 +50,8 @@ def _run(expedition_dir: str | Path, from_data: Path | None = None) -> None: print("╚═════════════════════════════════════════════════╝") if from_data is None: + # TODO: caution, if collaborative environments, will this mean everyone uses the same credentials file? + # TODO: may need to think about how to deal with this if using collaborative environments AND streaming data via copernicusmarine COPERNICUS_CREDS_FILE = os.path.expandvars( "$HOME/.copernicusmarine/.copernicusmarine-credentials" ) diff --git a/src/virtualship/cli/commands.py b/src/virtualship/cli/commands.py index f2f9f272d..3e485497c 100644 --- a/src/virtualship/cli/commands.py +++ b/src/virtualship/cli/commands.py @@ -2,13 +2,13 @@ import click -from virtualship import utils from virtualship.cli._plan import _plan from virtualship.cli._run import _run from virtualship.utils import ( COPERNICUSMARINE_BGC_VARIABLES, COPERNICUSMARINE_PHYS_VARIABLES, EXPEDITION, + get_example_expedition, mfp_to_yaml, ) @@ -58,7 +58,7 @@ def init(path, from_mfp): ) else: # Create a default example expedition YAML - expedition.write_text(utils.get_example_expedition()) + expedition.write_text(get_example_expedition()) click.echo(f"Created '{expedition.name}' at {path}.") @@ -90,7 +90,8 @@ def plan(path): help="Use pre-downloaded data, saved to disk, for expedition, instead of streaming directly via Copernicus Marine" "Assumes all data is stored in prescribed directory, and all variables (as listed below) are present." f"Required variables are: {set(COPERNICUSMARINE_PHYS_VARIABLES + COPERNICUSMARINE_BGC_VARIABLES)}" - "Assumes that variable names at least contain the standard Copernicus Marine variable name as a substring.", + "Assumes that variable names at least contain the standard Copernicus Marine variable name as a substring." + "Will also take the first file found containing the variable name substring. CAUTION if multiple files contain the same variable name substring.", ) def run(path, from_data): """Execute the expedition simulations.""" From 4f533c52bfbdcea23a174789873ec730ec374a72 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 11 Nov 2025 09:31:43 +0100 Subject: [PATCH 65/97] bug fixes for unnecessary copernicusmarine call when using pre-downloaded data --- src/virtualship/cli/_run.py | 8 ++++++-- src/virtualship/models/expedition.py | 3 +++ src/virtualship/utils.py | 5 +++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/virtualship/cli/_run.py b/src/virtualship/cli/_run.py index 353f6a1fa..cf83da7d1 100644 --- a/src/virtualship/cli/_run.py +++ b/src/virtualship/cli/_run.py @@ -63,7 +63,8 @@ def _run(expedition_dir: str | Path, from_data: Path | None = None) -> None: pass else: print( - "\nPlease enter your log in details for the Copernicus Marine Service (only necessary the first time you run VirtualShip). \n\nIf you have not registered yet, please do so at https://marine.copernicus.eu/.\n" + "\nPlease enter your log in details for the Copernicus Marine Service (only necessary the first time you run VirtualShip). \n\nIf you have not registered yet, please do so at https://marine.copernicus.eu/.\n\n" + "If you did not expect to see this message, and intended to use pre-downloaded data instead of streaming via Copernicus Marine, please use the '--from-data' option to specify the path to the data.\n" ) copernicusmarine.login() @@ -85,7 +86,10 @@ def _run(expedition_dir: str | Path, from_data: Path | None = None) -> None: print("\n---- WAYPOINT VERIFICATION ----") - expedition.schedule.verify(expedition.ship_config.ship_speed_knots) + expedition.schedule.verify( + expedition.ship_config.ship_speed_knots, + from_data=Path(from_data) if from_data else None, + ) # simulate the schedule schedule_results = simulate_schedule( diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index 0b82e0af6..96e591b30 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -2,6 +2,7 @@ import itertools from datetime import datetime, timedelta +from pathlib import Path from typing import TYPE_CHECKING import pydantic @@ -90,6 +91,7 @@ def verify( ignore_land_test: bool = False, *, check_space_time_region: bool = False, + from_data: Path | None = None, ) -> None: """ Verify the feasibility and correctness of the schedule's waypoints. @@ -134,6 +136,7 @@ def verify( bathymetry_field = _get_bathy_data( self.space_time_region, latlon_buffer=None, + from_data=from_data, ).bathymetry except Exception as e: raise ScheduleError( diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index 9f1a08e87..d44c528fc 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -448,8 +448,9 @@ def _get_bathy_data( end_datetime=space_time_region.time_range.end_time, coordinates_selection_method="outside", ) - bathymetry_variables = {"bathymetry": "deptho"} - bathymetry_dimensions = {"lon": "longitude", "lat": "latitude"} + bathymetry_variables = {"bathymetry": "deptho"} + bathymetry_dimensions = {"lon": "longitude", "lat": "latitude"} + return FieldSet.from_xarray_dataset( ds_bathymetry, bathymetry_variables, bathymetry_dimensions ) From d288ead03fdafded2b235ee74c73a30b4c15a6bf Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 11 Nov 2025 09:51:48 +0100 Subject: [PATCH 66/97] remove redundant tests --- tests/cli/test_cli.py | 23 +-- tests/cli/test_creds.py | 66 --------- tests/cli/test_fetch.py | 139 ------------------ .../test_do_expedition.py => cli/test_run.py} | 6 +- 4 files changed, 4 insertions(+), 230 deletions(-) delete mode 100644 tests/cli/test_creds.py delete mode 100644 tests/cli/test_fetch.py rename tests/{expedition/test_do_expedition.py => cli/test_run.py} (54%) diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 1b2c6e53a..a0d90d773 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -5,7 +5,7 @@ import xarray as xr from click.testing import CliRunner -from virtualship.cli.commands import fetch, init +from virtualship.cli.commands import init from virtualship.utils import EXPEDITION @@ -65,24 +65,3 @@ def test_init_existing_expedition(): with pytest.raises(FileExistsError): result = runner.invoke(init, ["."]) raise result.exception - - -@pytest.mark.parametrize( - "fetch_args", - [ - [".", "--username", "test"], - [".", "--password", "test"], - ], -) -@pytest.mark.usefixtures("copernicus_no_download") -def test_fetch_both_creds_via_cli(runner, fetch_args): - result = runner.invoke(fetch, fetch_args) - assert result.exit_code == 1 - assert "Both username and password" in result.exc_info[1].args[0] - - -@pytest.mark.usefixtures("copernicus_no_download") -def test_fetch(runner): - """Test the fetch command, but mock the downloads (and metadata interrogation).""" - result = runner.invoke(fetch, [".", "--username", "test", "--password", "test"]) - assert result.exit_code == 0 diff --git a/tests/cli/test_creds.py b/tests/cli/test_creds.py deleted file mode 100644 index 17ef20234..000000000 --- a/tests/cli/test_creds.py +++ /dev/null @@ -1,66 +0,0 @@ -import pydantic -import pytest - -from virtualship.cli._creds import CredentialFileError, Credentials - - -def test_load_credentials(tmp_file): - tmp_file.write_text( - """ - COPERNICUS_USERNAME: test_user - COPERNICUS_PASSWORD: test_password - """ - ) - - creds = Credentials.from_yaml(tmp_file) - assert creds.COPERNICUS_USERNAME == "test_user" - assert creds.COPERNICUS_PASSWORD == "test_password" - - -# parameterize with the contents of the file -@pytest.mark.parametrize( - "contents", - [ - pytest.param( - """ - INVALID_KEY: some_value - """, - id="invalid-key", - ), - pytest.param( - """ - # number not allowed, should be string (or quoted number) - USERNAME: 123 - """, - id="number-not-allowed", - ), - ], -) -def test_invalid_credentials(tmp_file, contents): - tmp_file.write_text(contents) - - with pytest.raises(pydantic.ValidationError): - Credentials.from_yaml(tmp_file) - - -def test_credentials_invalid_format(tmp_file): - tmp_file.write_text( - """ - INVALID_FORMAT_BUT_VALID_YAML - """ - ) - - with pytest.raises(CredentialFileError): - Credentials.from_yaml(tmp_file) - - -def test_rt_credentials(tmp_file): - """Test round-trip for credentials using Credentials.from_yaml() and Credentials.dump().""" - creds = Credentials( - COPERNICUS_USERNAME="test_user", COPERNICUS_PASSWORD="test_password" - ) - - creds.to_yaml(tmp_file) - creds_loaded = Credentials.from_yaml(tmp_file) - - assert creds == creds_loaded diff --git a/tests/cli/test_fetch.py b/tests/cli/test_fetch.py deleted file mode 100644 index 4c041dbe5..000000000 --- a/tests/cli/test_fetch.py +++ /dev/null @@ -1,139 +0,0 @@ -from pathlib import Path - -import numpy as np -import pytest -import xarray as xr -from pydantic import BaseModel - -from virtualship.cli._fetch import ( - DOWNLOAD_METADATA, - DownloadMetadata, - IncompleteDownloadError, - _fetch, - assert_complete_download, - complete_download, - create_hash, - filename_to_hash, - get_existing_download, - hash_model, - hash_to_filename, -) -from virtualship.models import Expedition -from virtualship.utils import EXPEDITION, get_example_expedition - - -@pytest.fixture -def copernicus_no_download(monkeypatch): - """Mock the copernicusmarine `subset` and `open_dataset` functions, approximating the reanalysis products.""" - - # mock for copernicusmarine.subset - def fake_download(output_filename, output_directory, **_): - Path(output_directory).joinpath(output_filename).touch() - - def fake_open_dataset(*args, **kwargs): - return xr.Dataset( - coords={ - "time": ( - "time", - [ - np.datetime64("2022-01-01"), - np.datetime64("2025-01-01"), - ], # mock up rough renanalysis period - ) - } - ) - - monkeypatch.setattr("virtualship.cli._fetch.copernicusmarine.subset", fake_download) - monkeypatch.setattr( - "virtualship.cli._fetch.copernicusmarine.open_dataset", fake_open_dataset - ) - yield - - -@pytest.fixture -def expedition(tmpdir): - out_path = tmpdir.join(EXPEDITION) - - with open(out_path, "w") as file: - file.write(get_example_expedition()) - - expedition = Expedition.from_yaml(out_path) - - return expedition - - -@pytest.mark.usefixtures("copernicus_no_download") -def test_fetch(expedition, tmpdir): - """Test the fetch command, but mock the download and dataset metadata interrogation.""" - _fetch(Path(tmpdir), "test", "test") - - -def test_create_hash(): - assert len(create_hash("correct-length")) == 8 - assert create_hash("same") == create_hash("same") - assert create_hash("unique1") != create_hash("unique2") - - -def test_hash_filename_roundtrip(): - hash_ = create_hash("test") - assert filename_to_hash(hash_to_filename(hash_)) == hash_ - - -def test_hash_model(): - class TestModel(BaseModel): - a: int - b: str - - hash_model(TestModel(a=0, b="b")) - - -def test_complete_download(tmp_path): - # Setup - DownloadMetadata(download_complete=False).to_yaml(tmp_path / DOWNLOAD_METADATA) - - complete_download(tmp_path) - - assert_complete_download(tmp_path) - - -def test_assert_complete_download_complete(tmp_path): - # Setup - DownloadMetadata(download_complete=True).to_yaml(tmp_path / DOWNLOAD_METADATA) - - assert_complete_download(tmp_path) - - -def test_assert_complete_download_incomplete(tmp_path): - # Setup - DownloadMetadata(download_complete=False).to_yaml(tmp_path / DOWNLOAD_METADATA) - - with pytest.raises(IncompleteDownloadError): - assert_complete_download(tmp_path) - - -def test_assert_complete_download_missing(tmp_path): - with pytest.raises(IncompleteDownloadError): - assert_complete_download(tmp_path) - - -@pytest.fixture -def existing_data_folder(tmp_path, monkeypatch): - # Setup - folders = [ - "YYYYMMDD_HHMMSS_hash", - "YYYYMMDD_HHMMSS_hash2", - "some-invalid-data-folder", - "YYYYMMDD_HHMMSS_hash3", - ] - data_folder = tmp_path - monkeypatch.setattr( - "virtualship.cli._fetch.assert_complete_download", lambda x: None - ) - for f in folders: - (data_folder / f).mkdir() - yield data_folder - - -def test_get_existing_download(existing_data_folder): - assert isinstance(get_existing_download(existing_data_folder, "hash"), Path) - assert get_existing_download(existing_data_folder, "missing-hash") is None diff --git a/tests/expedition/test_do_expedition.py b/tests/cli/test_run.py similarity index 54% rename from tests/expedition/test_do_expedition.py rename to tests/cli/test_run.py index 0dbcd99a4..be15961ca 100644 --- a/tests/expedition/test_do_expedition.py +++ b/tests/cli/test_run.py @@ -2,11 +2,11 @@ from pytest import CaptureFixture -from virtualship.expedition import do_expedition +from virtualship.cli import _run -def test_do_expedition(capfd: CaptureFixture) -> None: - do_expedition("expedition_dir", input_data=Path("expedition_dir/input_data")) +def test_run(capfd: CaptureFixture) -> None: + _run("expedition_dir", input_data=Path("expedition_dir/input_data")) out, _ = capfd.readouterr() assert "Your expedition has concluded successfully!" in out, ( "Expedition did not complete successfully." From dc2f4d903b9a3bc93d23fc4da58cff3d9410e124 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 11 Nov 2025 08:52:47 +0000 Subject: [PATCH 67/97] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/virtualship/instruments/adcp.py | 2 +- src/virtualship/instruments/argo_float.py | 2 +- src/virtualship/instruments/base.py | 2 +- src/virtualship/instruments/ctd.py | 2 +- src/virtualship/instruments/ctd_bgc.py | 2 +- src/virtualship/instruments/drifter.py | 2 +- src/virtualship/instruments/ship_underwater_st.py | 2 +- src/virtualship/instruments/xbt.py | 2 +- src/virtualship/utils.py | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index 8dca31a3e..c53dbc1cf 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -2,8 +2,8 @@ from typing import ClassVar import numpy as np - from parcels import ParticleSet, ScipyParticle, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.utils import ( diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index 8ba794ba2..098774712 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -4,7 +4,6 @@ from typing import ClassVar import numpy as np - from parcels import ( AdvectionRK4, JITParticle, @@ -12,6 +11,7 @@ StatusCode, Variable, ) + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 5c764cb3b..fe698231c 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -6,9 +6,9 @@ import copernicusmarine import xarray as xr +from parcels import FieldSet from yaspin import yaspin -from parcels import FieldSet from virtualship.errors import CopernicusCatalogueError from virtualship.utils import ( COPERNICUSMARINE_PHYS_VARIABLES, diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 04fb24edd..4684c822f 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -3,8 +3,8 @@ from typing import TYPE_CHECKING, ClassVar import numpy as np - from parcels import JITParticle, ParticleSet, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 59dc4bc23..d3adcec3f 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np - from parcels import JITParticle, ParticleSet, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index 8ad0fe197..3ee8ad1ca 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np - from parcels import AdvectionRK4, JITParticle, ParticleSet, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index a28ae12db..6d8d45753 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -2,8 +2,8 @@ from typing import ClassVar import numpy as np - from parcels import ParticleSet, ScipyParticle, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.utils import add_dummy_UV, register_instrument diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index 5972efffe..df68122cf 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np - from parcels import JITParticle, ParticleSet, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index d44c528fc..9474e5c48 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -11,8 +11,8 @@ import copernicusmarine import numpy as np import xarray as xr - from parcels import FieldSet + from virtualship.errors import CopernicusCatalogueError if TYPE_CHECKING: From 2a05ed45f597a799f1cc5ed34b04ecd85268d707 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 11 Nov 2025 11:36:45 +0100 Subject: [PATCH 68/97] update tests (not yet instrument subclasses) --- src/virtualship/instruments/base.py | 3 + src/virtualship/static/expedition.yaml | 14 +- tests/cli/test_run.py | 65 +++++++- tests/conftest.py | 2 +- tests/expedition/test_expedition.py | 12 +- tests/instruments/test_base.py | 218 ++++++++++++------------- tests/test_utils.py | 210 ++++++++++++++++++++++-- 7 files changed, 374 insertions(+), 150 deletions(-) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 5c764cb3b..2188332ae 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -22,6 +22,9 @@ from virtualship.models import Expedition +# TODO: from-data should default to None and only be overwritten if specified in `virtualship run` ... + + class Instrument(abc.ABC): """Base class for instruments and their simulation.""" diff --git a/src/virtualship/static/expedition.yaml b/src/virtualship/static/expedition.yaml index 256bee87c..2b7707350 100644 --- a/src/virtualship/static/expedition.yaml +++ b/src/virtualship/static/expedition.yaml @@ -8,8 +8,8 @@ schedule: minimum_depth: 0 maximum_depth: 2000 time_range: - start_time: 2023-01-01 00:00:00 - end_time: 2023-02-01 00:00:00 + start_time: 1998-01-01 00:00:00 + end_time: 1998-02-01 00:00:00 waypoints: - instrument: - CTD @@ -17,30 +17,30 @@ schedule: location: latitude: 0 longitude: 0 - time: 2023-01-01 00:00:00 + time: 1998-01-01 00:00:00 - instrument: - DRIFTER - CTD location: latitude: 0.01 longitude: 0.01 - time: 2023-01-01 01:00:00 + time: 1998-01-01 01:00:00 - instrument: - ARGO_FLOAT location: latitude: 0.02 longitude: 0.02 - time: 2023-01-01 02:00:00 + time: 1998-01-01 02:00:00 - instrument: - XBT location: latitude: 0.03 longitude: 0.03 - time: 2023-01-01 03:00:00 + time: 1998-01-01 03:00:00 - location: latitude: 0.03 longitude: 0.03 - time: 2023-01-01 03:00:00 + time: 1998-01-01 03:00:00 instruments_config: adcp_config: num_bins: 40 diff --git a/tests/cli/test_run.py b/tests/cli/test_run.py index be15961ca..daa822f01 100644 --- a/tests/cli/test_run.py +++ b/tests/cli/test_run.py @@ -1,13 +1,64 @@ +from datetime import datetime from pathlib import Path -from pytest import CaptureFixture +from virtualship.cli._run import _run +from virtualship.expedition.simulate_schedule import ( + MeasurementsToSimulate, + ScheduleOk, +) +from virtualship.utils import EXPEDITION, get_example_expedition -from virtualship.cli import _run +def _simulate_schedule(projection, expedition): + """Return a trivial ScheduleOk with no measurements to simulate.""" + return ScheduleOk( + time=datetime.now(), measurements_to_simulate=MeasurementsToSimulate() + ) + + +class DummyInstrument: + """Dummy instrument class that just creates empty output directories.""" + + def __init__(self, expedition, directory, from_data=None): + """Initialize DummyInstrument.""" + self.expedition = expedition + self.directory = Path(directory) + self.from_data = from_data + + def execute(self, measurements, out_path): + """Mock execute method.""" + out_path = Path(out_path) + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.mkdir(parents=True, exist_ok=True) + + +def test_run(tmp_path, monkeypatch): + """Testing as if using pre-downloaded, local data.""" + expedition_dir = tmp_path / "expedition_dir" + expedition_dir.mkdir() + (expedition_dir / EXPEDITION).write_text(get_example_expedition()) + + monkeypatch.setattr("virtualship.cli._run.simulate_schedule", _simulate_schedule) -def test_run(capfd: CaptureFixture) -> None: - _run("expedition_dir", input_data=Path("expedition_dir/input_data")) - out, _ = capfd.readouterr() - assert "Your expedition has concluded successfully!" in out, ( - "Expedition did not complete successfully." + monkeypatch.setattr( + "virtualship.models.InstrumentsConfig.verify", lambda self, expedition: None ) + monkeypatch.setattr( + "virtualship.models.Schedule.verify", lambda self, *args, **kwargs: None + ) + + monkeypatch.setattr( + "virtualship.cli._run.get_instrument_class", lambda itype: DummyInstrument + ) + + fake_data_dir = None + + _run(expedition_dir, from_data=fake_data_dir) + + results_dir = expedition_dir / "results" + + assert results_dir.exists() and results_dir.is_dir() + cost_file = results_dir / "cost.txt" + assert cost_file.exists() + content = cost_file.read_text() + assert "cost:" in content diff --git a/tests/conftest.py b/tests/conftest.py index 1b7a1de09..5ceac033c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,4 @@ -"""Test configuration that is ran for every test.""" +"""Test configuration that is run for every test.""" import pytest diff --git a/tests/expedition/test_expedition.py b/tests/expedition/test_expedition.py index 4bed35e70..d20211312 100644 --- a/tests/expedition/test_expedition.py +++ b/tests/expedition/test_expedition.py @@ -5,7 +5,6 @@ import pytest from virtualship.errors import InstrumentsConfigError, ScheduleError -from virtualship.expedition.do_expedition import _load_input_data from virtualship.models import ( Expedition, Location, @@ -57,7 +56,7 @@ def test_verify_schedule() -> None: ship_speed_knots = _get_expedition(expedition_dir).ship_config.ship_speed_knots - schedule.verify(ship_speed_knots, None) + schedule.verify(ship_speed_knots, ignore_land_test=True) def test_get_instruments() -> None: @@ -156,7 +155,7 @@ def test_get_instruments() -> None: ), True, ScheduleError, - "space_time_region not found in schedule, please define it to fetch the data.", + "space_time_region not found in schedule, please define it to proceed.", id="NoSpaceTimeRegion", ), ], @@ -165,16 +164,11 @@ def test_verify_schedule_errors( schedule: Schedule, check_space_time_region: bool, error, match ) -> None: expedition = _get_expedition(expedition_dir) - input_data = _load_input_data( - expedition_dir, - expedition, - input_data=Path("expedition_dir/input_data"), - ) with pytest.raises(error, match=match): schedule.verify( expedition.ship_config.ship_speed_knots, - input_data, + ignore_land_test=True, check_space_time_region=check_space_time_region, ) diff --git a/tests/instruments/test_base.py b/tests/instruments/test_base.py index 0a58efcd9..272290e67 100644 --- a/tests/instruments/test_base.py +++ b/tests/instruments/test_base.py @@ -1,119 +1,14 @@ -import datetime -from pathlib import Path from unittest.mock import MagicMock, patch -import numpy as np -import pytest -import xarray as xr - -from virtualship.instruments.base import InputDataset, Instrument +from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType -from virtualship.models.space_time_region import ( - SpaceTimeRegion, - SpatialRange, - TimeRange, -) -from virtualship.utils import get_input_dataset_class - -# test dataclass, particle class, kernels, etc. are defined for each instrument +from virtualship.utils import get_instrument_class -def test_all_instruments_have_input_class(): +def test_all_instruments_have_instrument_class(): for instrument in InstrumentType: - input_class = get_input_dataset_class(instrument) - assert input_class is not None, f"No input_class for {instrument}" - - -# test InputDataset class - - -class DummyInputDataset(InputDataset): - """A minimal InputDataset subclass for testing purposes.""" - - def get_datasets_dict(self): - """Return a dummy datasets dict for testing.""" - return { - "dummy": { - "physical": True, - "variables": ["var1"], - "output_filename": "dummy.nc", - } - } - - -@pytest.fixture -def dummy_space_time_region(): - spatial_range = SpatialRange( - minimum_longitude=0, - maximum_longitude=1, - minimum_latitude=0, - maximum_latitude=1, - minimum_depth=0, - maximum_depth=10, - ) - base_time = datetime.datetime.strptime("1950-01-01", "%Y-%m-%d") - time_range = TimeRange( - start_time=base_time, - end_time=base_time + datetime.timedelta(hours=1), - ) - return SpaceTimeRegion( - spatial_range=spatial_range, - time_range=time_range, - ) - - -def test_dummyinputdataset_initialization(dummy_space_time_region): - ds = DummyInputDataset( - name="test", - latlon_buffer=0.5, - datetime_buffer=1, - min_depth=0, - max_depth=10, - data_dir=".", - credentials={"username": "u", "password": "p"}, - space_time_region=dummy_space_time_region, - ) - assert ds.name == "test" - assert ds.latlon_buffer == 0.5 - assert ds.datetime_buffer == 1 - assert ds.min_depth == 0 - assert ds.max_depth == 10 - assert ds.data_dir == "." - assert ds.credentials["username"] == "u" - - -@patch("virtualship.instruments.base.copernicusmarine.open_dataset") -@patch("virtualship.instruments.base.copernicusmarine.subset") -def test_download_data_calls_subset( - mock_subset, mock_open_dataset, dummy_space_time_region -): - """Test that download_data calls the subset function correctly, will also test Copernicus Marine product id search logic.""" - mock_open_dataset.return_value = xr.Dataset( - { - "time": ( - "time", - [ - np.datetime64("1993-01-01T00:00:00"), - np.datetime64("2023-01-01T01:00:00"), - ], - ) - } - ) - ds = DummyInputDataset( - name="test", - latlon_buffer=0.5, - datetime_buffer=1, - min_depth=0, - max_depth=10, - data_dir=".", - credentials={"username": "u", "password": "p"}, - space_time_region=dummy_space_time_region, - ) - ds.download_data() - assert mock_subset.called - - -# test Instrument class + instrument_class = get_instrument_class(instrument) + assert instrument_class is not None, f"No instrument_class for {instrument}" class DummyInstrument(Instrument): @@ -125,16 +20,18 @@ def simulate(self, data_dir, measurements, out_path): @patch("virtualship.instruments.base.FieldSet") -@patch("virtualship.instruments.base.get_existing_download") -@patch("virtualship.instruments.base.get_space_time_region_hash") -def test_load_input_data_calls(mock_hash, mock_get_download, mock_FieldSet): +@patch( + "virtualship.instruments.base._select_product_id", return_value="dummy_product_id" +) +@patch("virtualship.instruments.base.copernicusmarine") +def test_load_input_data(mock_copernicusmarine, mock_select_product_id, mock_FieldSet): """Test Instrument.load_input_data with mocks.""" - mock_hash.return_value = "hash" - mock_get_download.return_value = Path("/tmp/data") mock_fieldset = MagicMock() mock_FieldSet.from_netcdf.return_value = mock_fieldset + mock_FieldSet.from_xarray_dataset.return_value = mock_fieldset mock_fieldset.gridset.grids = [MagicMock(negate_depth=MagicMock())] mock_fieldset.__getitem__.side_effect = lambda k: MagicMock() + mock_copernicusmarine.open_dataset.return_value = MagicMock() dummy = DummyInstrument( name="test", expedition=MagicMock(schedule=MagicMock(space_time_region=MagicMock())), @@ -144,7 +41,96 @@ def test_load_input_data_calls(mock_hash, mock_get_download, mock_FieldSet): add_bathymetry=False, allow_time_extrapolation=False, verbose_progress=False, + from_data=None, ) fieldset = dummy.load_input_data() - assert mock_FieldSet.from_netcdf.called + assert mock_FieldSet.from_xarray_dataset.called + assert fieldset == mock_fieldset assert fieldset == mock_fieldset + + +def test_execute_calls_simulate(monkeypatch): + dummy = DummyInstrument( + name="test", + expedition=MagicMock(schedule=MagicMock(space_time_region=MagicMock())), + directory="/tmp", + filenames={"A": "a.nc"}, + variables={"A": "a"}, + add_bathymetry=False, + allow_time_extrapolation=False, + verbose_progress=True, + from_data=None, + ) + dummy.simulate = MagicMock() + dummy.execute([1, 2, 3], "/tmp/out") + dummy.simulate.assert_called_once() + + +def test_get_spec_value_buffer_and_limit(): + dummy = DummyInstrument( + name="test", + expedition=MagicMock(schedule=MagicMock(space_time_region=MagicMock())), + directory="/tmp", + filenames={"A": "a.nc"}, + variables={"A": "a"}, + add_bathymetry=False, + allow_time_extrapolation=False, + verbose_progress=False, + buffer_spec={"latlon": 5.0}, + limit_spec={"depth_min": 10.0}, + from_data=None, + ) + assert dummy._get_spec_value("buffer", "latlon", 0.0) == 5.0 + assert dummy._get_spec_value("limit", "depth_min", None) == 10.0 + assert dummy._get_spec_value("buffer", "missing", 42) == 42 + + +def test_generate_fieldset_combines_fields(monkeypatch): + dummy = DummyInstrument( + name="test", + expedition=MagicMock(schedule=MagicMock(space_time_region=MagicMock())), + directory="/tmp", + filenames={"A": "a.nc", "B": "b.nc"}, + variables={"A": "a", "B": "b"}, + add_bathymetry=False, + allow_time_extrapolation=False, + verbose_progress=False, + from_data=None, + ) + dummy.from_data = None + + monkeypatch.setattr(dummy, "_get_copernicus_ds", lambda physical, var: MagicMock()) + + fs_A = MagicMock() + fs_B = MagicMock() + fs_B.B = MagicMock() + monkeypatch.setattr( + "virtualship.instruments.base.FieldSet.from_xarray_dataset", + lambda ds, varmap, dims, mesh=None: fs_A if "A" in varmap else fs_B, + ) + monkeypatch.setattr(fs_A, "add_field", MagicMock()) + dummy._generate_fieldset() + fs_A.add_field.assert_called_once_with(fs_B.B) + + +def test_load_input_data_error(monkeypatch): + dummy = DummyInstrument( + name="test", + expedition=MagicMock(schedule=MagicMock(space_time_region=MagicMock())), + directory="/tmp", + filenames={"A": "a.nc"}, + variables={"A": "a"}, + add_bathymetry=False, + allow_time_extrapolation=False, + verbose_progress=False, + from_data=None, + ) + monkeypatch.setattr( + dummy, "_generate_fieldset", lambda: (_ for _ in ()).throw(Exception("fail")) + ) + import virtualship.errors + + try: + dummy.load_input_data() + except virtualship.errors.CopernicusCatalogueError as e: + assert "Failed to load input data" in str(e) diff --git a/tests/test_utils.py b/tests/test_utils.py index bb8208f65..b53505408 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,5 +1,94 @@ +from pathlib import Path + +import numpy as np +import pytest +import xarray as xr + +import virtualship.utils +from parcels import FieldSet from virtualship.models.expedition import Expedition -from virtualship.utils import get_example_expedition +from virtualship.utils import ( + _find_nc_file_with_variable, + _get_bathy_data, + _select_product_id, + _start_end_in_product_timerange, + add_dummy_UV, + get_example_expedition, +) + + +@pytest.fixture +def expedition(tmp_file): + with open(tmp_file, "w") as file: + file.write(get_example_expedition()) + return Expedition.from_yaml(tmp_file) + + +@pytest.fixture +def dummy_spatial_range(): + class DummySpatialRange: + minimum_longitude = 0 + maximum_longitude = 1 + minimum_latitude = 0 + maximum_latitude = 1 + minimum_depth = 0 + maximum_depth = 4 + + return DummySpatialRange() + + +@pytest.fixture +def dummy_time_range(): + class DummyTimeRange: + start_time = "2020-01-01" + end_time = "2020-01-02" + + return DummyTimeRange() + + +@pytest.fixture +def dummy_space_time_region(dummy_spatial_range, dummy_time_range): + class DummySpaceTimeRegion: + spatial_range = dummy_spatial_range + time_range = dummy_time_range + + return DummySpaceTimeRegion() + + +@pytest.fixture +def dummy_instrument(): + class DummyInstrument: + pass + + return DummyInstrument() + + +@pytest.fixture +def copernicus_no_download(monkeypatch): + """Mock the copernicusmarine `subset` and `open_dataset` functions, approximating the reanalysis products.""" + + # mock for copernicusmarine.subset + def fake_download(output_filename, output_directory, **_): + Path(output_directory).joinpath(output_filename).touch() + + def fake_open_dataset(*args, **kwargs): + return xr.Dataset( + coords={ + "time": ( + "time", + [ + np.datetime64("1993-01-01"), + np.datetime64("2022-01-01"), + ], # mock up rough renanalysis period + ) + } + ) + + monkeypatch.setattr("virtualship.utils.copernicusmarine.subset", fake_download) + monkeypatch.setattr( + "virtualship.utils.copernicusmarine.open_dataset", fake_open_dataset + ) + yield def test_get_example_expedition(): @@ -14,17 +103,118 @@ def test_valid_example_expedition(tmp_path): Expedition.from_yaml(path) -def test_instrument_registry_updates(): +def test_instrument_registry_updates(dummy_instrument): from virtualship import utils - class DummyInputDataset: - pass + utils.register_instrument("DUMMY_TYPE")(dummy_instrument) - class DummyInstrument: - pass + assert utils.INSTRUMENT_CLASS_MAP["DUMMY_TYPE"] is dummy_instrument + + +def test_add_dummy_UV_adds_fields(): + fieldset = FieldSet.from_data({"T": 1}, {"lon": 0, "lat": 0}, mesh="spherical") + fieldset.__dict__.pop("U", None) + fieldset.__dict__.pop("V", None) + + # should not have U or V fields initially + assert "U" not in fieldset.__dict__ + assert "V" not in fieldset.__dict__ + + add_dummy_UV(fieldset) + + # now U and V should be present + assert "U" in fieldset.__dict__ + assert "V" in fieldset.__dict__ + + # should not raise error if U and V already present + add_dummy_UV(fieldset) + + +@pytest.mark.usefixtures("copernicus_no_download") +def test_select_product_id(expedition): + """Should return the physical reanalysis product id via the timings prescribed in the static schedule.yaml file.""" + result = _select_product_id( + physical=True, + schedule_start=expedition.schedule.space_time_region.time_range.start_time, + schedule_end=expedition.schedule.space_time_region.time_range.end_time, + username="test", + password="test", + ) + assert result == "cmems_mod_glo_phy_my_0.083deg_P1D-m" + + +@pytest.mark.usefixtures("copernicus_no_download") +def test_start_end_in_product_timerange(expedition): + """Should return True for valid range ass determined by the static schedule.yaml file.""" + assert _start_end_in_product_timerange( + selected_id="cmems_mod_glo_phy_my_0.083deg_P1D-m", + schedule_start=expedition.schedule.space_time_region.time_range.start_time, + schedule_end=expedition.schedule.space_time_region.time_range.end_time, + username="test", + password="test", + ) + + +def test_get_bathy_data_local(tmp_path, dummy_space_time_region): + """Test that _get_bathy_data returns a FieldSet when given a local directory for --from-data.""" + # dummy .nc file with 'deptho' variable + data = np.array([[1, 2], [3, 4]]) + ds = xr.Dataset( + { + "deptho": (("x", "y"), data), + }, + coords={ + "longitude": (("x", "y"), np.array([[0, 1], [0, 1]])), + "latitude": (("x", "y"), np.array([[0, 0], [1, 1]])), + }, + ) + nc_path = tmp_path / "dummy.nc" + ds.to_netcdf(nc_path) + + # should return a FieldSet + fieldset = _get_bathy_data(dummy_space_time_region, from_data=tmp_path) + assert isinstance(fieldset, FieldSet) + assert hasattr(fieldset, "bathymetry") + assert np.allclose(fieldset.bathymetry.data, data) + + +def test_get_bathy_data_copernicusmarine(monkeypatch, dummy_space_time_region): + """Test that _get_bathy_data calls copernicusmarine by default.""" + + def dummy_copernicusmarine(*args, **kwargs): + raise RuntimeError("copernicusmarine called") + + monkeypatch.setattr( + virtualship.utils.copernicusmarine, "open_dataset", dummy_copernicusmarine + ) + + try: + _get_bathy_data(dummy_space_time_region) + except RuntimeError as e: + assert "copernicusmarine called" in str(e) + + +def test_find_nc_file_with_variable_substring(tmp_path): + # dummy .nc file with variable 'uo_glor' (possible for CMS products to have similar suffixes...) + data = np.array([[1, 2], [3, 4]]) + ds = xr.Dataset( + { + "uo_glor": (("x", "y"), data), + }, + coords={ + "longitude": (("x", "y"), np.array([[0, 1], [0, 1]])), + "latitude": (("x", "y"), np.array([[0, 0], [1, 1]])), + }, + ) + nc_path = tmp_path / "test.nc" + ds.to_netcdf(nc_path) + + # should find 'uo_glor' when searching for 'uo' + result = _find_nc_file_with_variable(tmp_path, "uo") + assert result is not None + filename, found_var = result + assert filename == "test.nc" + assert found_var == "uo_glor" - utils.register_input_dataset("DUMMY_TYPE")(DummyInputDataset) - utils.register_instrument("DUMMY_TYPE")(DummyInstrument) - assert utils.INPUT_DATASET_MAP["DUMMY_TYPE"] is DummyInputDataset - assert utils.INSTRUMENT_CLASS_MAP["DUMMY_TYPE"] is DummyInstrument +# TODO: add test that pre-downloaded data is in correct directories - when have moved to be able to handle temporally separated .nc files! From 95a583d3e6ddef67f9b786f5895d492e33522f6d Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 11 Nov 2025 13:25:03 +0100 Subject: [PATCH 69/97] update instrument tests --- src/virtualship/instruments/base.py | 4 +- src/virtualship/instruments/xbt.py | 2 + tests/instruments/test_adcp.py | 27 ++++++----- tests/instruments/test_argo_float.py | 23 +++++----- tests/instruments/test_ctd.py | 47 +++++++++++++++----- tests/instruments/test_ctd_bgc.py | 23 +++++----- tests/instruments/test_drifter.py | 24 +++++----- tests/instruments/test_ship_underwater_st.py | 26 ++++++----- tests/instruments/test_xbt.py | 23 +++++----- 9 files changed, 124 insertions(+), 75 deletions(-) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 2188332ae..9996a670a 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -24,6 +24,8 @@ # TODO: from-data should default to None and only be overwritten if specified in `virtualship run` ... +# TODO: update CMS credentials automation workflow so not all using the same credentials if running in a Jupyter Collaborative Session...! + class Instrument(abc.ABC): """Base class for instruments and their simulation.""" @@ -105,7 +107,7 @@ def simulate( def execute(self, measurements: list, out_path: str | Path) -> None: """Run instrument simulation.""" # - TMP = True # temporary spinner implementation + TMP = False # temporary spinner implementation # if TMP: if not self.verbose_progress: diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index 5972efffe..b9fe604fb 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -150,6 +150,8 @@ def simulate(self, measurements, out_path) -> None: f"XBT max_depth or bathymetry shallower than minimum {-DT * fall_speed}. It is likely the XBT cannot be deployed in this area, which is too shallow." ) + breakpoint() + # define xbt particles xbt_particleset = ParticleSet( fieldset=fieldset, diff --git a/tests/instruments/test_adcp.py b/tests/instruments/test_adcp.py index 569f15a1f..0f9003d69 100644 --- a/tests/instruments/test_adcp.py +++ b/tests/instruments/test_adcp.py @@ -4,9 +4,9 @@ import numpy as np import xarray as xr -from parcels import FieldSet -from virtualship.instruments.adcp import simulate_adcp +from parcels import FieldSet +from virtualship.instruments.adcp import ADCPInstrument from virtualship.models import Location, Spacetime @@ -77,17 +77,22 @@ def test_simulate_adcp(tmpdir) -> None: }, ) - # perform simulation + # dummy expedition and directory for ADCPInstrument + class DummyExpedition: + class instruments_config: + class adcp_config: + max_depth_meter = MAX_DEPTH + num_bins = NUM_BINS + + expedition = DummyExpedition() + directory = tmpdir + from_data = None + + adcp_instrument = ADCPInstrument(expedition, directory, from_data) out_path = tmpdir.join("out.zarr") - simulate_adcp( - fieldset=fieldset, - out_path=out_path, - max_depth=MAX_DEPTH, - min_depth=MIN_DEPTH, - num_bins=NUM_BINS, - sample_points=sample_points, - ) + adcp_instrument.load_input_data = lambda: fieldset + adcp_instrument.simulate(sample_points, out_path) results = xr.open_zarr(out_path) diff --git a/tests/instruments/test_argo_float.py b/tests/instruments/test_argo_float.py index 3eda53ae4..045a7b7b9 100644 --- a/tests/instruments/test_argo_float.py +++ b/tests/instruments/test_argo_float.py @@ -4,9 +4,9 @@ import numpy as np import xarray as xr -from parcels import FieldSet -from virtualship.instruments.argo_float import ArgoFloat, simulate_argo_floats +from parcels import FieldSet +from virtualship.instruments.argo_float import ArgoFloat, ArgoFloatInstrument from virtualship.models import Location, Spacetime @@ -53,16 +53,19 @@ def test_simulate_argo_floats(tmpdir) -> None: ) ] - # perform simulation + # dummy expedition and directory for ArgoFloatInstrument + class DummyExpedition: + pass + + expedition = DummyExpedition() + directory = tmpdir + from_data = None + + argo_instrument = ArgoFloatInstrument(expedition, directory, from_data) out_path = tmpdir.join("out.zarr") - simulate_argo_floats( - fieldset=fieldset, - out_path=out_path, - argo_floats=argo_floats, - outputdt=timedelta(minutes=5), - endtime=None, - ) + argo_instrument.load_input_data = lambda: fieldset + argo_instrument.simulate(argo_floats, out_path) # test if output is as expected results = xr.open_zarr(out_path) diff --git a/tests/instruments/test_ctd.py b/tests/instruments/test_ctd.py index 14e0a2765..39b6cf47b 100644 --- a/tests/instruments/test_ctd.py +++ b/tests/instruments/test_ctd.py @@ -5,13 +5,12 @@ """ import datetime -from datetime import timedelta import numpy as np import xarray as xr -from parcels import Field, FieldSet -from virtualship.instruments.ctd import CTD, simulate_ctd +from parcels import Field, FieldSet +from virtualship.instruments.ctd import CTD, CTDInstrument from virtualship.models import Location, Spacetime @@ -102,15 +101,42 @@ def test_simulate_ctds(tmpdir) -> None: ) fieldset.add_field(Field("bathymetry", [-1000], lon=0, lat=0)) - # perform simulation + # dummy expedition and directory for CTDInstrument + class DummyExpedition: + class schedule: + class space_time_region: + time_range = type( + "TimeRange", + (), + { + "start_time": fieldset.T.grid.time_origin.fulltime( + fieldset.T.grid.time_full[0] + ), + "end_time": fieldset.T.grid.time_origin.fulltime( + fieldset.T.grid.time_full[-1] + ), + }, + )() + spatial_range = type( + "SpatialRange", + (), + { + "minimum_longitude": 0, + "maximum_longitude": 1, + "minimum_latitude": 0, + "maximum_latitude": 1, + }, + )() + + expedition = DummyExpedition() + directory = tmpdir + from_data = None + + ctd_instrument = CTDInstrument(expedition, directory, from_data) out_path = tmpdir.join("out.zarr") - simulate_ctd( - ctds=ctds, - fieldset=fieldset, - out_path=out_path, - outputdt=timedelta(seconds=10), - ) + ctd_instrument.load_input_data = lambda: fieldset + ctd_instrument.simulate(ctds, out_path) # test if output is as expected results = xr.open_zarr(out_path) @@ -132,6 +158,7 @@ def test_simulate_ctds(tmpdir) -> None: for var in ["salinity", "temperature", "lat", "lon"]: obs_value = obs[var].values.item() exp_value = exp[var] + assert np.isclose(obs_value, exp_value), ( f"Observation incorrect {ctd_i=} {loc=} {var=} {obs_value=} {exp_value=}." ) diff --git a/tests/instruments/test_ctd_bgc.py b/tests/instruments/test_ctd_bgc.py index c12138842..4495f3e06 100644 --- a/tests/instruments/test_ctd_bgc.py +++ b/tests/instruments/test_ctd_bgc.py @@ -5,13 +5,12 @@ """ import datetime -from datetime import timedelta import numpy as np import xarray as xr -from parcels import Field, FieldSet -from virtualship.instruments.ctd_bgc import CTD_BGC, simulate_ctd_bgc +from parcels import Field, FieldSet +from virtualship.instruments.ctd_bgc import CTD_BGC, CTD_BGCInstrument from virtualship.models import Location, Spacetime @@ -163,15 +162,19 @@ def test_simulate_ctd_bgcs(tmpdir) -> None: ) fieldset.add_field(Field("bathymetry", [-1000], lon=0, lat=0)) - # perform simulation + # dummy expedition and directory for CTD_BGCInstrument + class DummyExpedition: + pass + + expedition = DummyExpedition() + directory = tmpdir + from_data = None + + ctd_bgc_instrument = CTD_BGCInstrument(expedition, directory, from_data) out_path = tmpdir.join("out.zarr") - simulate_ctd_bgc( - ctd_bgcs=ctd_bgcs, - fieldset=fieldset, - out_path=out_path, - outputdt=timedelta(seconds=10), - ) + ctd_bgc_instrument.load_input_data = lambda: fieldset + ctd_bgc_instrument.simulate(ctd_bgcs, out_path) # test if output is as expected results = xr.open_zarr(out_path) diff --git a/tests/instruments/test_drifter.py b/tests/instruments/test_drifter.py index ae230a873..095e6cdfe 100644 --- a/tests/instruments/test_drifter.py +++ b/tests/instruments/test_drifter.py @@ -4,9 +4,9 @@ import numpy as np import xarray as xr -from parcels import FieldSet -from virtualship.instruments.drifter import Drifter, simulate_drifters +from parcels import FieldSet +from virtualship.instruments.drifter import Drifter, DrifterInstrument from virtualship.models import Location, Spacetime @@ -52,17 +52,19 @@ def test_simulate_drifters(tmpdir) -> None: ), ] - # perform simulation + # dummy expedition and directory for DrifterInstrument + class DummyExpedition: + pass + + expedition = DummyExpedition() + directory = tmpdir + from_data = None + + drifter_instrument = DrifterInstrument(expedition, directory, from_data) out_path = tmpdir.join("out.zarr") - simulate_drifters( - fieldset=fieldset, - out_path=out_path, - drifters=drifters, - outputdt=datetime.timedelta(hours=1), - dt=datetime.timedelta(minutes=5), - endtime=None, - ) + drifter_instrument.load_input_data = lambda: fieldset + drifter_instrument.simulate(drifters, out_path) # test if output is as expected results = xr.open_zarr(out_path) diff --git a/tests/instruments/test_ship_underwater_st.py b/tests/instruments/test_ship_underwater_st.py index 9d44ee6df..8e1cfbdc0 100644 --- a/tests/instruments/test_ship_underwater_st.py +++ b/tests/instruments/test_ship_underwater_st.py @@ -4,16 +4,13 @@ import numpy as np import xarray as xr -from parcels import FieldSet -from virtualship.instruments.ship_underwater_st import simulate_ship_underwater_st +from parcels import FieldSet +from virtualship.instruments.ship_underwater_st import Underwater_STInstrument from virtualship.models import Location, Spacetime def test_simulate_ship_underwater_st(tmpdir) -> None: - # depth at which the sampling will be done - DEPTH = -2 - # arbitrary time offset for the dummy fieldset base_time = datetime.datetime.strptime("1950-01-01", "%Y-%m-%d") @@ -70,15 +67,20 @@ def test_simulate_ship_underwater_st(tmpdir) -> None: }, ) - # perform simulation + # dummy expedition and directory for Underwater_STInstrument + class DummyExpedition: + pass + + expedition = DummyExpedition() + directory = tmpdir + from_data = None + + st_instrument = Underwater_STInstrument(expedition, directory, from_data) out_path = tmpdir.join("out.zarr") - simulate_ship_underwater_st( - fieldset=fieldset, - out_path=out_path, - depth=DEPTH, - sample_points=sample_points, - ) + st_instrument.load_input_data = lambda: fieldset + # The instrument expects measurements as sample_points + st_instrument.simulate(sample_points, out_path) # test if output is as expected results = xr.open_zarr(out_path) diff --git a/tests/instruments/test_xbt.py b/tests/instruments/test_xbt.py index 97e33ade8..7bafef9f4 100644 --- a/tests/instruments/test_xbt.py +++ b/tests/instruments/test_xbt.py @@ -5,13 +5,12 @@ """ import datetime -from datetime import timedelta import numpy as np import xarray as xr -from parcels import Field, FieldSet -from virtualship.instruments.xbt import XBT, simulate_xbt +from parcels import Field, FieldSet +from virtualship.instruments.xbt import XBT, XBTInstrument from virtualship.models import Location, Spacetime @@ -96,15 +95,19 @@ def test_simulate_xbts(tmpdir) -> None: ) fieldset.add_field(Field("bathymetry", [-1000], lon=0, lat=0)) - # perform simulation + # dummy expedition and directory for XBTInstrument + class DummyExpedition: + pass + + expedition = DummyExpedition() + directory = tmpdir + from_data = None + + xbt_instrument = XBTInstrument(expedition, directory, from_data) out_path = tmpdir.join("out.zarr") - simulate_xbt( - xbts=xbts, - fieldset=fieldset, - out_path=out_path, - outputdt=timedelta(seconds=10), - ) + xbt_instrument.load_input_data = lambda: fieldset + xbt_instrument.simulate(xbts, out_path) # test if output is as expected results = xr.open_zarr(out_path) From 411c0427a8d9aff9245e5d0469870641ffe03fbe Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 11 Nov 2025 14:42:24 +0100 Subject: [PATCH 70/97] test is on land tests in schedule.verify() --- src/virtualship/instruments/base.py | 26 +++------ src/virtualship/instruments/xbt.py | 2 - src/virtualship/models/expedition.py | 11 +++- tests/expedition/test_expedition.py | 82 ++++++++++++++++++++++++++++ 4 files changed, 99 insertions(+), 22 deletions(-) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 9996a670a..4eb8d0425 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -106,26 +106,18 @@ def simulate( def execute(self, measurements: list, out_path: str | Path) -> None: """Run instrument simulation.""" - # - TMP = False # temporary spinner implementation - # - if TMP: - if not self.verbose_progress: - with yaspin( - text=f"Simulating {self.name} measurements... ", - side="right", - spinner=ship_spinner, - ) as spinner: - self.simulate(measurements, out_path) - spinner.ok("✅\n") - else: - print(f"Simulating {self.name} measurements... ") + if not self.verbose_progress: + with yaspin( + text=f"Simulating {self.name} measurements... ", + side="right", + spinner=ship_spinner, + ) as spinner: self.simulate(measurements, out_path) - print("\n") - - # + spinner.ok("✅\n") else: + print(f"Simulating {self.name} measurements... ") self.simulate(measurements, out_path) + print("\n") def _get_copernicus_ds( self, diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index b9fe604fb..5972efffe 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -150,8 +150,6 @@ def simulate(self, measurements, out_path) -> None: f"XBT max_depth or bathymetry shallower than minimum {-DT * fall_speed}. It is likely the XBT cannot be deployed in this area, which is too shallow." ) - breakpoint() - # define xbt particles xbt_particleset = ParticleSet( fieldset=fieldset, diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index 96e591b30..18846c1c3 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -5,6 +5,7 @@ from pathlib import Path from typing import TYPE_CHECKING +import numpy as np import pydantic import pyproj import yaml @@ -145,14 +146,18 @@ def verify( for wp_i, wp in enumerate(self.waypoints): try: - bathymetry_field.eval( + value = bathymetry_field.eval( 0, # time 0, # depth (surface) wp.location.lat, wp.location.lon, ) - except Exception: - land_waypoints.append((wp_i, wp)) + if value == 0.0 or (isinstance(value, float) and np.isnan(value)): + land_waypoints.append((wp_i, wp)) + except Exception as e: + raise ScheduleError( + f"Waypoint #{wp_i + 1} at location {wp.location} could not be evaluated against bathymetry data. There may be a problem with the waypoint location being outside of the space_time_region or with the bathymetry data itself.\n\n Original error: {e}" + ) from e if len(land_waypoints) > 0: raise ScheduleError( diff --git a/tests/expedition/test_expedition.py b/tests/expedition/test_expedition.py index d20211312..9e08ecfce 100644 --- a/tests/expedition/test_expedition.py +++ b/tests/expedition/test_expedition.py @@ -1,9 +1,13 @@ from datetime import datetime, timedelta from pathlib import Path +from unittest.mock import patch +import numpy as np import pyproj import pytest +import xarray as xr +from parcels import FieldSet from virtualship.errors import InstrumentsConfigError, ScheduleError from virtualship.models import ( Expedition, @@ -11,6 +15,11 @@ Schedule, Waypoint, ) +from virtualship.models.space_time_region import ( + SpaceTimeRegion, + SpatialRange, + TimeRange, +) from virtualship.utils import EXPEDITION, _get_expedition, get_example_expedition projection = pyproj.Geod(ellps="WGS84") @@ -85,6 +94,79 @@ def test_get_instruments() -> None: ) +def test_verify_on_land(): + """Test that schedule verification raises error for waypoints on land (0.0 m bathymetry).""" + # bathymetry fieldset with NaNs at specific locations + latitude = np.array([0, 1.0, 2.0]) + longitude = np.array([0, 1.0, 2.0]) + bathymetry = np.array( + [ + [100, 0.0, 100], + [100, 100, 0.0], + [0.0, 100, 100], + ] + ) + + ds_bathymetry = xr.Dataset( + { + "deptho": (("latitude", "longitude"), bathymetry), + }, + coords={ + "latitude": latitude, + "longitude": longitude, + }, + ) + + bathymetry_variables = {"bathymetry": "deptho"} + bathymetry_dimensions = {"lon": "longitude", "lat": "latitude"} + bathymetry_fieldset = FieldSet.from_xarray_dataset( + ds_bathymetry, bathymetry_variables, bathymetry_dimensions + ) + + # waypoints placed in NaN bathy cells + waypoints = [ + Waypoint( + location=Location(0.0, 1.0), time=datetime(2022, 1, 1, 1, 0, 0) + ), # NaN cell + Waypoint( + location=Location(1.0, 2.0), time=datetime(2022, 1, 2, 1, 0, 0) + ), # NaN cell + Waypoint( + location=Location(2.0, 0.0), time=datetime(2022, 1, 3, 1, 0, 0) + ), # NaN cell + ] + + spatial_range = SpatialRange( + minimum_latitude=min(wp.location.lat for wp in waypoints), + maximum_latitude=max(wp.location.lat for wp in waypoints), + minimum_longitude=min(wp.location.lon for wp in waypoints), + maximum_longitude=max(wp.location.lon for wp in waypoints), + ) + time_range = TimeRange( + start_time=min(wp.time for wp in waypoints if wp.time is not None), + end_time=max(wp.time for wp in waypoints if wp.time is not None), + ) + space_time_region = SpaceTimeRegion( + spatial_range=spatial_range, time_range=time_range + ) + schedule = Schedule(waypoints=waypoints, space_time_region=space_time_region) + ship_speed_knots = _get_expedition(expedition_dir).ship_config.ship_speed_knots + + with patch( + "virtualship.models.expedition._get_bathy_data", + return_value=bathymetry_fieldset, + ): + with pytest.raises( + ScheduleError, + match=r"The following waypoint\(s\) throw\(s\) error\(s\):", + ): + schedule.verify( + ship_speed_knots, + ignore_land_test=False, + from_data=None, + ) + + @pytest.mark.parametrize( "schedule,check_space_time_region,error,match", [ From e364d00cbdda77757f8b8a26afab59632da6cdd3 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 11 Nov 2025 14:52:11 +0100 Subject: [PATCH 71/97] Run pre-commit --- src/virtualship/instruments/adcp.py | 2 +- src/virtualship/instruments/argo_float.py | 2 +- src/virtualship/instruments/base.py | 2 +- src/virtualship/instruments/ctd.py | 2 +- src/virtualship/instruments/ctd_bgc.py | 2 +- src/virtualship/instruments/drifter.py | 2 +- src/virtualship/instruments/ship_underwater_st.py | 2 +- src/virtualship/instruments/xbt.py | 2 +- src/virtualship/utils.py | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index c53dbc1cf..8dca31a3e 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -2,8 +2,8 @@ from typing import ClassVar import numpy as np -from parcels import ParticleSet, ScipyParticle, Variable +from parcels import ParticleSet, ScipyParticle, Variable from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.utils import ( diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index 098774712..8ba794ba2 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -4,6 +4,7 @@ from typing import ClassVar import numpy as np + from parcels import ( AdvectionRK4, JITParticle, @@ -11,7 +12,6 @@ StatusCode, Variable, ) - from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index ccec22949..4eb8d0425 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -6,9 +6,9 @@ import copernicusmarine import xarray as xr -from parcels import FieldSet from yaspin import yaspin +from parcels import FieldSet from virtualship.errors import CopernicusCatalogueError from virtualship.utils import ( COPERNICUSMARINE_PHYS_VARIABLES, diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 4684c822f..04fb24edd 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -3,8 +3,8 @@ from typing import TYPE_CHECKING, ClassVar import numpy as np -from parcels import JITParticle, ParticleSet, Variable +from parcels import JITParticle, ParticleSet, Variable from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index d3adcec3f..59dc4bc23 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np -from parcels import JITParticle, ParticleSet, Variable +from parcels import JITParticle, ParticleSet, Variable from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index 3ee8ad1ca..8ad0fe197 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np -from parcels import AdvectionRK4, JITParticle, ParticleSet, Variable +from parcels import AdvectionRK4, JITParticle, ParticleSet, Variable from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index 6d8d45753..a28ae12db 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -2,8 +2,8 @@ from typing import ClassVar import numpy as np -from parcels import ParticleSet, ScipyParticle, Variable +from parcels import ParticleSet, ScipyParticle, Variable from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.utils import add_dummy_UV, register_instrument diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index df68122cf..5972efffe 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np -from parcels import JITParticle, ParticleSet, Variable +from parcels import JITParticle, ParticleSet, Variable from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index 9474e5c48..d44c528fc 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -11,8 +11,8 @@ import copernicusmarine import numpy as np import xarray as xr -from parcels import FieldSet +from parcels import FieldSet from virtualship.errors import CopernicusCatalogueError if TYPE_CHECKING: From 56d41e6a4fdfd9939c2dada67ebd9a2379be0b39 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 12 Nov 2025 09:32:04 +0100 Subject: [PATCH 72/97] update pre-download ingestion methods to take files split by time --- src/virtualship/instruments/base.py | 102 +++++++++++++++------------ src/virtualship/models/expedition.py | 5 -- src/virtualship/utils.py | 8 ++- 3 files changed, 60 insertions(+), 55 deletions(-) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 4eb8d0425..3ca1cdfad 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -1,6 +1,7 @@ import abc +import re from collections import OrderedDict -from datetime import timedelta +from datetime import datetime, timedelta from pathlib import Path from typing import TYPE_CHECKING @@ -156,51 +157,11 @@ def _get_copernicus_ds( coordinates_selection_method="outside", ) - def _load_local_ds(self, filename) -> xr.Dataset: - """ - Load local dataset from specified data directory. - - Sliced according to expedition.schedule.space_time_region and buffer specs. - """ - ds = xr.open_dataset(self.from_data.joinpath(filename)) - - coord_rename = {} - if "lat" in ds.coords: - coord_rename["lat"] = "latitude" - if "lon" in ds.coords: - coord_rename["lon"] = "longitude" - if coord_rename: - ds = ds.rename(coord_rename) - - min_lon = ( - self.expedition.schedule.space_time_region.spatial_range.minimum_longitude - - self._get_spec_value( - "buffer", "latlon", 3.0 - ) # always add min 3 deg buffer for local data to avoid edge issues with ds.sel() - ) - max_lon = ( - self.expedition.schedule.space_time_region.spatial_range.maximum_longitude - + self._get_spec_value("buffer", "latlon", 3.0) - ) - min_lat = ( - self.expedition.schedule.space_time_region.spatial_range.minimum_latitude - - self._get_spec_value("buffer", "latlon", 3.0) - ) - max_lat = ( - self.expedition.schedule.space_time_region.spatial_range.maximum_latitude - + self._get_spec_value("buffer", "latlon", 3.0) - ) - - return ds.sel( - latitude=slice(min_lat, max_lat), - longitude=slice(min_lon, max_lon), - ) - def _generate_fieldset(self) -> FieldSet: """ Create and combine FieldSets for each variable, supporting both local and Copernicus Marine data sources. - Avoids issues when using copernicusmarine and creating directly one FieldSet of ds's sourced from different Copernicus Marine product IDs, which is often the case for BGC variables. + Per variable avoids issues when using copernicusmarine and creating directly one FieldSet of ds's sourced from different Copernicus Marine product IDs, which is often the case for BGC variables. """ fieldsets_list = [] keys = list(self.variables.keys()) @@ -208,12 +169,34 @@ def _generate_fieldset(self) -> FieldSet: for key in keys: var = self.variables[key] if self.from_data is not None: # load from local data - filename, full_var_name = _find_nc_file_with_variable( - self.from_data, var + physical = var in COPERNICUSMARINE_PHYS_VARIABLES + if physical: + data_dir = self.from_data.joinpath("phys") + else: + data_dir = self.from_data.joinpath("bgc") + + schedule_start = ( + self.expedition.schedule.space_time_region.time_range.start_time ) - ds = self._load_local_ds(filename) - fs = FieldSet.from_xarray_dataset( - ds, {key: full_var_name}, self.dimensions, mesh="spherical" + schedule_end = ( + self.expedition.schedule.space_time_region.time_range.end_time + ) + + files = self._find_files_in_timerange( + data_dir, + schedule_start, + schedule_end, + ) + + _, full_var_name = _find_nc_file_with_variable( + data_dir, var + ) # get full variable name from one of the files; var may only appear as substring in variable name in file + + fs = FieldSet.from_netcdf( + filenames=[data_dir.joinpath(f) for f in files], + variables={key: full_var_name}, + dimensions=self.dimensions, + mesh="spherical", ) else: # steam via Copernicus Marine physical = var in COPERNICUSMARINE_PHYS_VARIABLES @@ -233,3 +216,28 @@ def _get_spec_value(self, spec_type: str, key: str, default=None): """Helper to extract a value from buffer_spec or limit_spec.""" spec = self.buffer_spec if spec_type == "buffer" else self.limit_spec return spec.get(key) if spec and spec.get(key) is not None else default + + def _find_files_in_timerange( + self, + data_dir: Path, + schedule_start, + schedule_end, + date_pattern=r"\d{4}_\d{2}_\d{2}", + date_fmt="%Y_%m_%d", + ) -> list: + """Find all files in data_dir whose filenames contain a date within [schedule_start, schedule_end] (inclusive).""" + # TODO: scope to make this more flexible for different date patterns / formats ... + files_with_dates = [] + start_date = schedule_start.date() # normalise to date only for comparison (given start/end dates have hour/minute components which may exceed those in file_date) + end_date = schedule_end.date() + for file in data_dir.iterdir(): + if file.is_file(): + match = re.search(date_pattern, file.name) + if match: + file_date = datetime.strptime(match.group(), date_fmt).date() + if start_date <= file_date <= end_date: + files_with_dates.append((file_date, file.name)) + files_with_dates.sort( + key=lambda x: x[0] + ) # sort by extracted date; more robust than relying on filesystem order + return [fname for _, fname in files_with_dates] diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index 18846c1c3..1ca5076d5 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -3,7 +3,6 @@ import itertools from datetime import datetime, timedelta from pathlib import Path -from typing import TYPE_CHECKING import numpy as np import pydantic @@ -17,10 +16,6 @@ from .location import Location from .space_time_region import SpaceTimeRegion -if TYPE_CHECKING: - pass - - projection: pyproj.Geod = pyproj.Geod(ellps="WGS84") diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index d44c528fc..e0ca1dc09 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -419,13 +419,15 @@ def _get_bathy_data( """Bathymetry data from local or 'streamed' directly from Copernicus Marine.""" if from_data is not None: # load from local data var = "deptho" + bathy_dir = from_data.joinpath("bathymetry/") try: - filename, _ = _find_nc_file_with_variable(from_data, var) + filename, _ = _find_nc_file_with_variable(bathy_dir, var) except Exception as e: + # TODO: link to documentation on expected data structure!! raise RuntimeError( - f"Could not find bathymetry variable '{var}' in provided data directory '{from_data}'." + f"\n\n❗️ Could not find bathymetry variable '{var}' in data directory '{from_data}/bathymetry/'.\n\n❗️ Is the pre-downloaded data directory structure compliant with VirtualShip expectations?\n\n❗️ See for more information on expectations: <<>>\n" ) from e - ds_bathymetry = xr.open_dataset(from_data.joinpath(filename)) + ds_bathymetry = xr.open_dataset(bathy_dir.joinpath(filename)) bathymetry_variables = {"bathymetry": "deptho"} bathymetry_dimensions = {"lon": "longitude", "lat": "latitude"} return FieldSet.from_xarray_dataset( From 845a8f8916c1f71b44d9dc1f876c025f29ac4407 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 13 Nov 2025 15:36:29 +0100 Subject: [PATCH 73/97] fix bug --- tests/test_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index b53505408..46bb1b78f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -168,7 +168,8 @@ def test_get_bathy_data_local(tmp_path, dummy_space_time_region): "latitude": (("x", "y"), np.array([[0, 0], [1, 1]])), }, ) - nc_path = tmp_path / "dummy.nc" + nc_path = tmp_path / "bathymetry/dummy.nc" + nc_path.parent.mkdir(parents=True, exist_ok=True) ds.to_netcdf(nc_path) # should return a FieldSet From 1418a446a7aa39829cb1a7512646ba828be84679 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 13 Nov 2025 15:36:48 +0100 Subject: [PATCH 74/97] fix bug in ingesting bgc data from disk --- src/virtualship/instruments/base.py | 81 +++++++++++++++++++++++------ 1 file changed, 66 insertions(+), 15 deletions(-) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 3ca1cdfad..423910525 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -1,4 +1,5 @@ import abc +import glob import re from collections import OrderedDict from datetime import datetime, timedelta @@ -107,18 +108,24 @@ def simulate( def execute(self, measurements: list, out_path: str | Path) -> None: """Run instrument simulation.""" - if not self.verbose_progress: - with yaspin( - text=f"Simulating {self.name} measurements... ", - side="right", - spinner=ship_spinner, - ) as spinner: + TMP = True + + if TMP: + if not self.verbose_progress: + with yaspin( + text=f"Simulating {self.name} measurements... ", + side="right", + spinner=ship_spinner, + ) as spinner: + self.simulate(measurements, out_path) + spinner.ok("✅\n") + else: + print(f"Simulating {self.name} measurements... ") self.simulate(measurements, out_path) - spinner.ok("✅\n") + print("\n") + else: - print(f"Simulating {self.name} measurements... ") self.simulate(measurements, out_path) - print("\n") def _get_copernicus_ds( self, @@ -192,8 +199,11 @@ def _generate_fieldset(self) -> FieldSet: data_dir, var ) # get full variable name from one of the files; var may only appear as substring in variable name in file - fs = FieldSet.from_netcdf( - filenames=[data_dir.joinpath(f) for f in files], + ds = xr.open_mfdataset( + [data_dir.joinpath(f) for f in files] + ) # as ds --> .from_xarray_dataset seems more robust than .from_netcdf for handling different temporal resolutions for different variables ... + fs = FieldSet.from_xarray_dataset( + ds, variables={key: full_var_name}, dimensions=self.dimensions, mesh="spherical", @@ -227,17 +237,58 @@ def _find_files_in_timerange( ) -> list: """Find all files in data_dir whose filenames contain a date within [schedule_start, schedule_end] (inclusive).""" # TODO: scope to make this more flexible for different date patterns / formats ... + + all_files = glob.glob(str(data_dir.joinpath("*"))) + if not all_files: + raise ValueError( + f"No files found in data directory {data_dir}. Please ensure the directory contains files with 'P1D' or 'P1M' in their names as per Copernicus Marine Product ID naming conventions." + ) + + if all("P1D" in s for s in all_files): + t_resolution = "daily" + elif all("P1M" in s for s in all_files): + t_resolution = "monthly" + else: + raise ValueError( + f"Could not determine time resolution from filenames in data directory. Please ensure all filenames in {data_dir} contain either 'P1D' (daily) or 'P1M' (monthly), " + f"as per the Copernicus Marine Product ID naming conventions." + ) + + if t_resolution == "monthly": + t_min = schedule_start.date() + t_max = ( + schedule_end.date() + + timedelta( + days=32 + ) # buffer to ensure fieldset end date is always longer than schedule end date for monthly data + ) + else: # daily + t_min = schedule_start.date() + t_max = schedule_end.date() + files_with_dates = [] - start_date = schedule_start.date() # normalise to date only for comparison (given start/end dates have hour/minute components which may exceed those in file_date) - end_date = schedule_end.date() for file in data_dir.iterdir(): if file.is_file(): match = re.search(date_pattern, file.name) if match: - file_date = datetime.strptime(match.group(), date_fmt).date() - if start_date <= file_date <= end_date: + file_date = datetime.strptime( + match.group(), date_fmt + ).date() # normalise to date only for comparison (given start/end dates have hour/minute components which may exceed those in file_date) + if t_min <= file_date <= t_max: files_with_dates.append((file_date, file.name)) + files_with_dates.sort( key=lambda x: x[0] ) # sort by extracted date; more robust than relying on filesystem order + + # catch if not enough data coverage found for the requested time range + if files_with_dates[-1][0] < schedule_end.date(): + raise ValueError( + f"Not enough data coverage found in {data_dir} for the requested time range {schedule_start} to {schedule_end}. " + f"Latest available data is for date {files_with_dates[-1][0]}." + f"If using monthly data, please ensure that the last month downloaded covers the schedule end date + 1 month." + f"See documentation for more details: <>" + # TODO: add link to relevant documentation! + ) + return [fname for _, fname in files_with_dates] From 5226a0257b66d4f90e01886b9956ead4e5b11eef Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 13 Nov 2025 16:38:16 +0100 Subject: [PATCH 75/97] tidy up --- docs/user-guide/quickstart.md | 2 -- src/virtualship/cli/_run.py | 2 +- src/virtualship/cli/commands.py | 1 - src/virtualship/cli/validator_utils.py | 1 - .../expedition/simulate_schedule.py | 3 +- src/virtualship/instruments/base.py | 34 +++++++------------ src/virtualship/models/expedition.py | 1 - 7 files changed, 14 insertions(+), 30 deletions(-) diff --git a/docs/user-guide/quickstart.md b/docs/user-guide/quickstart.md index 32dc0fdc4..f4f232ad1 100644 --- a/docs/user-guide/quickstart.md +++ b/docs/user-guide/quickstart.md @@ -151,5 +151,3 @@ It might take up to an hour to simulate the measurements depending on your choic Upon successfully completing the simulation, results from the expedition will be stored in the `EXPEDITION_NAME/results` directory, written as [Zarr](https://zarr.dev/) files. From here you can carry on your analysis (offline). We encourage you to explore and analyse these data using [Xarray](https://docs.xarray.dev/en/stable/). We also provide various further [VirtualShip tutorials](https://virtualship.readthedocs.io/en/latest/user-guide/tutorials/index.html) which provide examples of how to visualise data recorded by the VirtualShip instruments. - - diff --git a/src/virtualship/cli/_run.py b/src/virtualship/cli/_run.py index cf83da7d1..06337cf3e 100644 --- a/src/virtualship/cli/_run.py +++ b/src/virtualship/cli/_run.py @@ -51,7 +51,7 @@ def _run(expedition_dir: str | Path, from_data: Path | None = None) -> None: if from_data is None: # TODO: caution, if collaborative environments, will this mean everyone uses the same credentials file? - # TODO: may need to think about how to deal with this if using collaborative environments AND streaming data via copernicusmarine + # TODO: need to think about how to deal with this for when using collaborative environments AND streaming data via copernicusmarine COPERNICUS_CREDS_FILE = os.path.expandvars( "$HOME/.copernicusmarine/.copernicusmarine-credentials" ) diff --git a/src/virtualship/cli/commands.py b/src/virtualship/cli/commands.py index 3e485497c..b5840f174 100644 --- a/src/virtualship/cli/commands.py +++ b/src/virtualship/cli/commands.py @@ -77,7 +77,6 @@ def plan(path): _plan(Path(path)) -# TODO: also add option to 'stream' via link to dir elsewhere, e.g. simlink or path to data stored elsewhere that isn't expedition dir! @click.command() @click.argument( "path", diff --git a/src/virtualship/cli/validator_utils.py b/src/virtualship/cli/validator_utils.py index 83239ac8d..402e48b18 100644 --- a/src/virtualship/cli/validator_utils.py +++ b/src/virtualship/cli/validator_utils.py @@ -123,7 +123,6 @@ def make_validator(condition, reference, value_type): Therefore, reference values for the conditions cannot be fed in dynamically and necessitates 'hard-coding' the condition and reference value combination. At present, Pydantic models in VirtualShip only require gt/ge/lt/le relative to **0.0** so the 'reference' value is always checked as being == 0.0 Additional custom conditions can be 'hard-coded' as new condition and reference combinations if Pydantic model specifications change in the future and/or new instruments are added to VirtualShip etc. - TODO: Perhaps there's scope here though for a more flexible implementation in a future PR... """ diff --git a/src/virtualship/expedition/simulate_schedule.py b/src/virtualship/expedition/simulate_schedule.py index 3b8fa78a1..1d43dc13b 100644 --- a/src/virtualship/expedition/simulate_schedule.py +++ b/src/virtualship/expedition/simulate_schedule.py @@ -122,8 +122,7 @@ def simulate(self) -> ScheduleOk | ScheduleProblem: # check if waypoint was reached in time if waypoint.time is not None and self._time > waypoint.time: print( - # TODO: I think this should be wp_i + 1, not wp_i; otherwise it will be off by one - f"Waypoint {wp_i} could not be reached in time. Current time: {self._time}. Waypoint time: {waypoint.time}." + f"Waypoint {wp_i + 1} could not be reached in time. Current time: {self._time}. Waypoint time: {waypoint.time}." ) return ScheduleProblem(self._time, wp_i) else: diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 423910525..54b45822a 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -24,11 +24,6 @@ from virtualship.models import Expedition -# TODO: from-data should default to None and only be overwritten if specified in `virtualship run` ... - -# TODO: update CMS credentials automation workflow so not all using the same credentials if running in a Jupyter Collaborative Session...! - - class Instrument(abc.ABC): """Base class for instruments and their simulation.""" @@ -108,24 +103,18 @@ def simulate( def execute(self, measurements: list, out_path: str | Path) -> None: """Run instrument simulation.""" - TMP = True - - if TMP: - if not self.verbose_progress: - with yaspin( - text=f"Simulating {self.name} measurements... ", - side="right", - spinner=ship_spinner, - ) as spinner: - self.simulate(measurements, out_path) - spinner.ok("✅\n") - else: - print(f"Simulating {self.name} measurements... ") + if not self.verbose_progress: + with yaspin( + text=f"Simulating {self.name} measurements... ", + side="right", + spinner=ship_spinner, + ) as spinner: self.simulate(measurements, out_path) - print("\n") - + spinner.ok("✅\n") else: + print(f"Simulating {self.name} measurements... ") self.simulate(measurements, out_path) + print("\n") def _get_copernicus_ds( self, @@ -201,7 +190,8 @@ def _generate_fieldset(self) -> FieldSet: ds = xr.open_mfdataset( [data_dir.joinpath(f) for f in files] - ) # as ds --> .from_xarray_dataset seems more robust than .from_netcdf for handling different temporal resolutions for different variables ... + ) # using: ds --> .from_xarray_dataset seems more robust than .from_netcdf for handling different temporal resolutions for different variables ... + fs = FieldSet.from_xarray_dataset( ds, variables={key: full_var_name}, @@ -236,7 +226,7 @@ def _find_files_in_timerange( date_fmt="%Y_%m_%d", ) -> list: """Find all files in data_dir whose filenames contain a date within [schedule_start, schedule_end] (inclusive).""" - # TODO: scope to make this more flexible for different date patterns / formats ... + # TODO: scope to make this more flexible for different date patterns / formats ... ? all_files = glob.glob(str(data_dir.joinpath("*"))) if not all_files: diff --git a/src/virtualship/models/expedition.py b/src/virtualship/models/expedition.py index 1ca5076d5..4847b10c4 100644 --- a/src/virtualship/models/expedition.py +++ b/src/virtualship/models/expedition.py @@ -125,7 +125,6 @@ def verify( ) # check if all waypoints are in water using bathymetry data - # TODO: write test that checks that will flag when waypoint is on land!! [add to existing suite of fail .verify() tests in test_expedition.py] land_waypoints = [] if not ignore_land_test: try: From 36574b0ce0dd9f24887b4d41681c73ecf37afc68 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 13 Nov 2025 16:48:38 +0100 Subject: [PATCH 76/97] add test for data directory structure compliance --- src/virtualship/utils.py | 2 +- tests/test_utils.py | 27 ++++++++++++++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index e0ca1dc09..e4a134035 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -419,7 +419,7 @@ def _get_bathy_data( """Bathymetry data from local or 'streamed' directly from Copernicus Marine.""" if from_data is not None: # load from local data var = "deptho" - bathy_dir = from_data.joinpath("bathymetry/") + bathy_dir = from_data.joinpath("bathymetry") try: filename, _ = _find_nc_file_with_variable(bathy_dir, var) except Exception as e: diff --git a/tests/test_utils.py b/tests/test_utils.py index 46bb1b78f..87e402bbf 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -218,4 +218,29 @@ def test_find_nc_file_with_variable_substring(tmp_path): assert found_var == "uo_glor" -# TODO: add test that pre-downloaded data is in correct directories - when have moved to be able to handle temporally separated .nc files! +def test_data_dir_structure_compliance(): + """ + Test that Instrument._generate_fieldset and _get_bathy_data use the expected subdirectory names. + + ('phys', 'bgc', 'bathymetry') for local data loading, as required by documentation. + + To avoid drift from what expecations are laid out in the docs. + """ + base_path = Path(__file__).parent.parent / "src/virtualship/instruments/base.py" + utils_path = Path(__file__).parent.parent / "src/virtualship/utils.py" + + base_code = base_path.read_text(encoding="utf-8") + utils_code = utils_path.read_text(encoding="utf-8") + + # Check for phys and bgc in Instrument._generate_fieldset + assert 'self.from_data.joinpath("phys")' in base_code, ( + "Expected 'phys' subdirectory not found in Instrument._generate_fieldset." + ) + assert 'self.from_data.joinpath("bgc")' in base_code, ( + "Expected 'bgc' subdirectory not found in Instrument._generate_fieldset." + ) + + # Check for bathymetry in _get_bathy_data + assert 'from_data.joinpath("bathymetry")' in utils_code, ( + "Expected 'bathymetry' subdirectory not found in _get_bathy_data." + ) From 4d9f5edf7f9ee3296e6e452ebe2b224d358399e5 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 13 Nov 2025 18:15:12 +0100 Subject: [PATCH 77/97] update docs --- .../example_copernicus_download.ipynb | 193 ++++++++++++++++++ .../documentation/pre_download_data.md | 69 +++++++ docs/user-guide/index.md | 2 + docs/user-guide/quickstart.md | 8 + 4 files changed, 272 insertions(+) create mode 100644 docs/user-guide/documentation/example_copernicus_download.ipynb create mode 100644 docs/user-guide/documentation/pre_download_data.md diff --git a/docs/user-guide/documentation/example_copernicus_download.ipynb b/docs/user-guide/documentation/example_copernicus_download.ipynb new file mode 100644 index 000000000..e57404050 --- /dev/null +++ b/docs/user-guide/documentation/example_copernicus_download.ipynb @@ -0,0 +1,193 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a48322c9", + "metadata": {}, + "source": [ + "# Example Copernicus data download \n", + "\n", + "This Notebook provides a rough example of how to download Copernicus Marine data using the Copernicus Marine API.\n", + "\n", + "This will download:\n", + "- Global bathymetry data (static)\n", + "- Global biogeochemical monthly data (0.25 degree hindcast)\n", + "- Global physical daily data (0.25 degree reanalysis)\n", + "\n", + "For a singular year (2023) and two months (June and July).\n", + "\n", + "This notebook is intended as a basic example only. Modifications will be needed to adapt this to your own use case." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7f5a7cc7", + "metadata": {}, + "outputs": [], + "source": [ + "import copernicusmarine\n", + "import os\n", + "from datetime import datetime" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e7279d5a", + "metadata": {}, + "outputs": [], + "source": [ + "YEAR = \"2023\"\n", + "MONTHS = [\"06\", \"07\"]\n", + "DAYS = [\n", + " \"01\",\n", + " \"02\",\n", + " \"03\",\n", + " \"04\",\n", + " \"05\",\n", + " \"06\",\n", + " \"07\",\n", + " \"08\",\n", + " \"09\",\n", + " \"10\",\n", + " \"11\",\n", + " \"12\",\n", + " \"13\",\n", + " \"14\",\n", + " \"15\",\n", + " \"16\",\n", + " \"17\",\n", + " \"18\",\n", + " \"19\",\n", + " \"20\",\n", + " \"21\",\n", + " \"22\",\n", + " \"23\",\n", + " \"24\",\n", + " \"25\",\n", + " \"26\",\n", + " \"27\",\n", + " \"28\",\n", + " \"29\",\n", + " \"30\",\n", + " \"31\",\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1a583dba", + "metadata": {}, + "outputs": [], + "source": [ + "### PHYSICAL DAILY FILES\n", + "\n", + "os.chdir(\"~/data/phys/\")\n", + "DATASET_ID = \"cmems_mod_glo_phy-all_my_0.25deg_P1D-m\"\n", + "\n", + "for month in MONTHS:\n", + " for day in DAYS:\n", + " # check is valid date\n", + " try:\n", + " datetime(year=int(YEAR), month=int(month), day=int(day), hour=0)\n", + " except ValueError:\n", + " continue\n", + "\n", + " filename = f\"{DATASET_ID}_global_fulldepth_{YEAR}_{month}_{day}.nc\"\n", + "\n", + " if os.path.exists(filename):\n", + " print(f\"File {filename} already exists, skipping...\")\n", + " continue\n", + "\n", + " copernicusmarine.subset(\n", + " dataset_id=DATASET_ID,\n", + " variables=[\"uo_glor\", \"vo_glor\", \"thetao_glor\", \"so_glor\"],\n", + " minimum_longitude=-180,\n", + " maximum_longitude=179.75,\n", + " minimum_latitude=-80,\n", + " maximum_latitude=90,\n", + " start_datetime=f\"{YEAR}-{month}-{day}T00:00:00\",\n", + " end_datetime=f\"{YEAR}-{month}-{day}T00:00:00\",\n", + " minimum_depth=0.5057600140571594,\n", + " maximum_depth=5902.0576171875,\n", + " output_filename=filename,\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "89921772", + "metadata": {}, + "outputs": [], + "source": [ + "### BIOGEOCHEMICAL MONTHLY FILES\n", + "\n", + "os.chdir(\"~/data/bgc/\")\n", + "DATASET_ID = \"cmems_mod_glo_bgc_my_0.25deg_P1M-m\"\n", + "DAY = \"01\"\n", + "\n", + "for month in MONTHS:\n", + " try:\n", + " datetime(year=int(YEAR), month=int(month), day=int(DAY), hour=0)\n", + " except ValueError:\n", + " continue\n", + "\n", + " filename = f\"{DATASET_ID}_global_fulldepth_{YEAR}_{month}_{DAY}.nc\"\n", + "\n", + " if os.path.exists(filename):\n", + " print(f\"File {filename} already exists, skipping...\")\n", + " continue\n", + "\n", + " copernicusmarine.subset(\n", + " dataset_id=\"cmems_mod_glo_bgc_my_0.25deg_P1M-m\",\n", + " variables=[\"chl\", \"no3\", \"nppv\", \"o2\", \"ph\", \"phyc\", \"po4\"],\n", + " minimum_longitude=-180,\n", + " maximum_longitude=179.75,\n", + " minimum_latitude=-80,\n", + " maximum_latitude=90,\n", + " start_datetime=f\"{YEAR}-{month}-{DAY}T00:00:00\",\n", + " end_datetime=f\"{YEAR}-{month}-{DAY}T00:00:00\",\n", + " minimum_depth=0.5057600140571594,\n", + " maximum_depth=5902.05810546875,\n", + " output_filename=filename,\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8b5495c6", + "metadata": {}, + "outputs": [], + "source": [ + "### BATHYMETRY FILE\n", + "os.chdir(\"~/data/bathymetry/\")\n", + "DATASET_ID = \"cmems_mod_glo_phy_anfc_0.083deg_static\"\n", + "filename = \"cmems_mod_glo_phy_anfc_0.083deg_static_bathymetry.nc\"\n", + "\n", + "copernicusmarine.subset(\n", + " dataset_id=DATASET_ID,\n", + " dataset_part=\"bathy\",\n", + " variables=[\"deptho\"],\n", + " minimum_longitude=-180,\n", + " maximum_longitude=179.91668701171875,\n", + " minimum_latitude=-80,\n", + " maximum_latitude=90,\n", + " minimum_depth=0.49402499198913574,\n", + " maximum_depth=0.49402499198913574,\n", + " output_filename=filename,\n", + ")" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/user-guide/documentation/pre_download_data.md b/docs/user-guide/documentation/pre_download_data.md new file mode 100644 index 000000000..a1877443a --- /dev/null +++ b/docs/user-guide/documentation/pre_download_data.md @@ -0,0 +1,69 @@ +# Pre-downloading data + +By default, VirtualShip will automatically 'stream' data from the Copernicus Marine Service via their [copernicusmarine toolbox](https://github.com/mercator-ocean/copernicus-marine-toolbox?tab=readme-ov-file). However, for users with limited or unreliable internet connectivity, or those wishing to manage data locally, it is possible to pre-download the required datasets. + +As outlined in the [Quickstart Guide](../tutorials/quickstart.md), the `virtualship run` command supports an optional `--from-data` argument, which allows users to specify a local directory containing the necessary data files. + +Example Python code for automating data download from Copernicus Marine can be found in the [Example Copernicus Download Notebook](example_copernicus_download.ipynb). + +### Data requirements + +When using pre-downloaded data with VirtualShip, the software supports only: daily and monthly resolution physical and biogeochemical data, along with a static bathymetry file. + +In addition, all pre-downloaded data must be split into separate files per timestep (i.e. one .nc file per day or month). + +Further, VirtualShip expects pre-downloaded data to be organised in a specific directory & filename structure within the specified local data directory. The expected structure is as outlined in the subsequent sections. + +#### Directory structure + +Assuming the local data directory (as supplied in the `--from-data` argument) is named `data/`, the expected subdirectory structure is: + +```bash +. +└── data + ├── bathymetry # containing the singular bathymetry .nc file + ├── bgc # containing biogeochemical .nc files + └── phys # containing physical .nc files +``` + +#### Filename conventions + +Within these subdirectories, the expected filename conventions are: + +- Physical data files (in `data/phys/`) should be named as follows: + - `_.nc` + - e.g. `cmems_mod_glo_phy-all_my_0.25deg_P1D-m_1998_05_01.nc` +- Biogeochemical data files (in `data/bgc/`) should be named as follows: + - `_.nc` + - e.g. `cmems_mod_glo_bgc_my_0.25deg_P1M-m_1998_05_01.nc` +- Bathymetry data file (in `data/bathymetry/`) should be named as follows: + - `cmems_mod_glo_phy_anfc_0.083deg_static_bathymetry.nc` + +```{tip} +Careful to use an underscore (`_`) as the separator between date components in the filenames (i.e. `YYYY_MM_DD`). +``` + +```{note} +Using the `` in the filenames is vital in order to correctly identify the temporal resolution of the data (daily or monthly). The `P1D` in the example above indicates daily data, whereas `P1M` would indicate monthly data. + +See [here](https://help.marine.copernicus.eu/en/articles/6820094-how-is-the-nomenclature-of-copernicus-marine-data-defined#h_34a5a6f21d) for more information on Copernicus dataset nomenclature. + +See also our own [documentation](copernicus_products.md) on the Copernicus products used natively by VirtualShip when streaming data. +``` + +#### Further assumptions + +The following assumptions are also made about the data: + +1. All pre-downloaded data files must be in NetCDF format (`.nc`). +2. Physical data files must contain the following variables: `uo`, `vo`, `so`, `thetao` + - Or these strings must appear as substrings within the variable names (e.g. `uo_glor` is acceptable for `uo`). +3. If using BGC instruments (e.g. `CTD_BGC`), the relevant biogeochemical data files must contain the following variables: `o2`, `chl`, `no3`, `po4`, `nppv`, `ph`, `phyc`. + - Or these strings must appear as substrings within the variable names (e.g. `o2_glor` is acceptable for `o2`). +4. Bathymetry data files must contain a variable named `deptho`. + +#### Also of note + +1. Whilst not mandatory to use data downloaded only from Copernicus Marine (any existing data you may hold can be re-organised accordingly), the assumptions made by VirtualShip regarding directory structure and filename conventions are motivated by alignment with the Copernicus Marine's practices. + - If you want to use pre-existing data with VirtualShip, which you may have accessed from a different source, it is possible to do so by restructuring and/or renaming your data files as necessary. +2. The whole VirtualShip pre-downloaded data workflow should support global data or subsets thereof, provided the data files contain the necessary variables and are structured as outlined above. diff --git a/docs/user-guide/index.md b/docs/user-guide/index.md index 2578fbea6..fac1c26cf 100644 --- a/docs/user-guide/index.md +++ b/docs/user-guide/index.md @@ -15,4 +15,6 @@ assignments/index :maxdepth: 1 documentation/copernicus_products.md +documentation/pre_download_data.md +documentation/example_copernicus_download.ipynb ``` diff --git a/docs/user-guide/quickstart.md b/docs/user-guide/quickstart.md index f4f232ad1..b2cec51ee 100644 --- a/docs/user-guide/quickstart.md +++ b/docs/user-guide/quickstart.md @@ -1,5 +1,13 @@ # VirtualShip Quickstart Guide 🚢 +```{warning} +This quickstart guide is currently out of date with the latest version of VirtualShip (v1.0.0). + +It will be updated soon. + +In particular, the `virtualship fetch` command is no longer supported. Instead, data fetching is now integrated into the `virtualship run` command. See [#226](https://github.com/Parcels-code/virtualship/pull/226) for more details in the meantime. +``` + Welcome to this Quickstart to using VirtualShip. In this guide we will conduct a virtual expedition in the North Sea. Note, however, that you can plan your own expedition anywhere in the global ocean and conduct whatever set of measurements you wish! This Quickstart is available as an instructional video below, or you can continue with the step-by-step guide. From e9f3b1647d9e511c76b3d6aef2fcade2da424648 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 13 Nov 2025 21:46:20 +0100 Subject: [PATCH 78/97] edits to docs --- .../documentation/copernicus_products.md | 10 +++---- .../example_copernicus_download.ipynb | 2 +- .../documentation/pre_download_data.md | 30 +++++++++++++++---- docs/user-guide/quickstart.md | 6 ++-- 4 files changed, 32 insertions(+), 16 deletions(-) diff --git a/docs/user-guide/documentation/copernicus_products.md b/docs/user-guide/documentation/copernicus_products.md index 78361984e..fec42d442 100644 --- a/docs/user-guide/documentation/copernicus_products.md +++ b/docs/user-guide/documentation/copernicus_products.md @@ -2,7 +2,7 @@ VirtualShip supports running experiments anywhere in the global ocean from 1993 through to the present day (and approximately two weeks into the future), using the suite of products available from the [Copernicus Marine Data Store](https://data.marine.copernicus.eu/products). -The data sourcing task is handled by the `virtualship fetch` command. The three products relied on by `fetch` to source data for all [VirtualShip instruments](https://virtualship.readthedocs.io/en/latest/user-guide/assignments/Research_proposal_intro.html#Measurement-Options) (both physical and biogeochemical) are: +The data sourcing task is handled by the `virtualship run` command, which in turn relies on the [copernicusmarine toolbox](https://github.com/mercator-ocean/copernicus-marine-toolbox?tab=readme-ov-file) for 'streaming' data from the Copernicus Marine Data Store. The three products relied on in `run` to source data for all [VirtualShip instruments](https://virtualship.readthedocs.io/en/latest/user-guide/assignments/Research_proposal_intro.html#Measurement-Options) (both physical and biogeochemical) are: 1. **Reanalysis** (or "hindcast" for biogeochemistry). 2. **Renalysis interim** (or "hindcast interim" for biogeochemistry). @@ -15,7 +15,7 @@ The Copernicus Marine Service describe the differences between the three product As a general rule of thumb the three different products span different periods across the historical period to present and are intended to allow for continuity across the previous ~ 30 years. ```{note} -The ethos for automated dataset selection in `virtualship fetch` is to prioritise the Reanalysis/Hindcast products where possible (the 'work horse'), then _interim products where possible for continuity, and finally filling the very near-present (and near-future) temporal range with the Analysis & Forecast products. +The ethos for automated dataset selection in `virtualship run` is to prioritise the Reanalysis/Hindcast products where possible (the 'work horse'), then _interim products where possible for continuity, and finally filling the very near-present (and near-future) temporal range with the Analysis & Forecast products. ``` ```{warning} @@ -24,13 +24,13 @@ In the rare situation where the start and end times of an expedition schedule sp ### Data availability -The following tables summarise which Copernicus product is selected by `virtualship fetch` per combination of time period and variable (see legend below). +The following tables summarise which Copernicus product is selected by `virtualship run` per combination of time period and variable (see legend below). For biogeochemical variables `ph` and `phyc`, monthly products are required for hindcast and hindcast interim periods. For all other variables, daily products are available. #### Physical products -| Period | Product ID | Temporal Resolution | Typical Years Covered | Variables | +| Period | Dataset ID | Temporal Resolution | Typical Years Covered | Variables | | :------------------ | :--------------------------------------- | :------------------ | :---------------------------------- | :------------------------- | | Reanalysis | `cmems_mod_glo_phy_my_0.083deg_P1D-m` | Daily | ~30 years ago to ~5 years ago | `uo`, `vo`, `so`, `thetao` | | Reanalysis Interim | `cmems_mod_glo_phy_myint_0.083deg_P1D-m` | Daily | ~5 years ago to ~2 months ago | `uo`, `vo`, `so`, `thetao` | @@ -40,7 +40,7 @@ For biogeochemical variables `ph` and `phyc`, monthly products are required for #### Biogeochemical products -| Period | Product ID | Temporal Resolution | Typical Years Covered | Variables | Notes | +| Period | Dataset ID | Temporal Resolution | Typical Years Covered | Variables | Notes | | :---------------------------- | :----------------------------------------- | :------------------ | :---------------------------------- | :-------------------------------- | :------------------------------------- | | Hindcast | `cmems_mod_glo_bgc_my_0.25deg_P1D-m` | Daily | ~30 years ago to ~5 years ago | `o2`, `chl`, `no3`, `po4`, `nppv` | Most BGC variables except `ph`, `phyc` | | Hindcast (monthly) | `cmems_mod_glo_bgc_my_0.25deg_P1M-m` | Monthly | ~30 years ago to ~5 years ago | `ph`, `phyc` | Only `ph`, `phyc` (monthly only) | diff --git a/docs/user-guide/documentation/example_copernicus_download.ipynb b/docs/user-guide/documentation/example_copernicus_download.ipynb index e57404050..1e630520f 100644 --- a/docs/user-guide/documentation/example_copernicus_download.ipynb +++ b/docs/user-guide/documentation/example_copernicus_download.ipynb @@ -7,7 +7,7 @@ "source": [ "# Example Copernicus data download \n", "\n", - "This Notebook provides a rough example of how to download Copernicus Marine data using the Copernicus Marine API.\n", + "This notebook provides a rough, non-optimised example of how to download Copernicus Marine data using the `copernicusmarine` Python package.\n", "\n", "This will download:\n", "- Global bathymetry data (static)\n", diff --git a/docs/user-guide/documentation/pre_download_data.md b/docs/user-guide/documentation/pre_download_data.md index a1877443a..f8d7e46aa 100644 --- a/docs/user-guide/documentation/pre_download_data.md +++ b/docs/user-guide/documentation/pre_download_data.md @@ -1,10 +1,14 @@ # Pre-downloading data -By default, VirtualShip will automatically 'stream' data from the Copernicus Marine Service via their [copernicusmarine toolbox](https://github.com/mercator-ocean/copernicus-marine-toolbox?tab=readme-ov-file). However, for users with limited or unreliable internet connectivity, or those wishing to manage data locally, it is possible to pre-download the required datasets. +By default, VirtualShip will automatically 'stream' data from the Copernicus Marine Service via the [copernicusmarine toolbox](https://github.com/mercator-ocean/copernicus-marine-toolbox?tab=readme-ov-file). However, for users who wish to manage data locally, it is possible to pre-download the required datasets and feed them into VirtualShip simulations. -As outlined in the [Quickstart Guide](../tutorials/quickstart.md), the `virtualship run` command supports an optional `--from-data` argument, which allows users to specify a local directory containing the necessary data files. + -Example Python code for automating data download from Copernicus Marine can be found in the [Example Copernicus Download Notebook](example_copernicus_download.ipynb). +As outlined in the [Quickstart Guide](https://virtualship.readthedocs.io/en/latest/user-guide/quickstart.html), the `virtualship run` command supports an optional `--from-data` argument, which allows users to specify a local directory containing the necessary data files. + +```{tip} +See the [for example...](#for-example) section for an example data download workflow. +``` ### Data requirements @@ -12,6 +16,10 @@ When using pre-downloaded data with VirtualShip, the software supports only: dai In addition, all pre-downloaded data must be split into separate files per timestep (i.e. one .nc file per day or month). +```{note} +**Monthly data**: when using monthly data, ensure that your final .nc file download is for the month *after* your expedition schedule end date. This is to ensure that a Parcels FieldSet can be generated under-the-hood which fully covers the expedition period. For example, if your expedition runs from 1st May to 15th May, your final monthly data file should be in June. Daily data files only need to cover the expedition period exactly. +``` + Further, VirtualShip expects pre-downloaded data to be organised in a specific directory & filename structure within the specified local data directory. The expected structure is as outlined in the subsequent sections. #### Directory structure @@ -32,10 +40,10 @@ Within these subdirectories, the expected filename conventions are: - Physical data files (in `data/phys/`) should be named as follows: - `_.nc` - - e.g. `cmems_mod_glo_phy-all_my_0.25deg_P1D-m_1998_05_01.nc` + - e.g. `cmems_mod_glo_phy-all_my_0.25deg_P1D-m_1998_05_01.nc` and so on for each timestep. - Biogeochemical data files (in `data/bgc/`) should be named as follows: - `_.nc` - - e.g. `cmems_mod_glo_bgc_my_0.25deg_P1M-m_1998_05_01.nc` + - e.g. `cmems_mod_glo_bgc_my_0.25deg_P1M-m_1998_05_01.nc` and so on for each timestep. - Bathymetry data file (in `data/bathymetry/`) should be named as follows: - `cmems_mod_glo_phy_anfc_0.083deg_static_bathymetry.nc` @@ -48,7 +56,11 @@ Using the `` in the filenames is vital in order See [here](https://help.marine.copernicus.eu/en/articles/6820094-how-is-the-nomenclature-of-copernicus-marine-data-defined#h_34a5a6f21d) for more information on Copernicus dataset nomenclature. -See also our own [documentation](copernicus_products.md) on the Copernicus products used natively by VirtualShip when streaming data. +See also our own [documentation](https://virtualship.readthedocs.io/en/latest/user-guide/documentation/copernicus_products.html#data-availability) on the Copernicus datasets used natively by VirtualShip when 'streaming' data if you wish to use the same datasets for pre-download. +``` + +```{note} +**Monthly data**: the `DD` component of the date in the filename for monthly .nc files should always be `01`, representing the first day of the month. This ensures that a Parcels FieldSet can be generated under-the-hood which fully covers the expedition period from the start. ``` #### Further assumptions @@ -67,3 +79,9 @@ The following assumptions are also made about the data: 1. Whilst not mandatory to use data downloaded only from Copernicus Marine (any existing data you may hold can be re-organised accordingly), the assumptions made by VirtualShip regarding directory structure and filename conventions are motivated by alignment with the Copernicus Marine's practices. - If you want to use pre-existing data with VirtualShip, which you may have accessed from a different source, it is possible to do so by restructuring and/or renaming your data files as necessary. 2. The whole VirtualShip pre-downloaded data workflow should support global data or subsets thereof, provided the data files contain the necessary variables and are structured as outlined above. + +#### For example... + +Example Python code for automating the data download from Copernicus Marine can be found in [Example Copernicus Download](example_copernicus_download.ipynb). + + diff --git a/docs/user-guide/quickstart.md b/docs/user-guide/quickstart.md index b2cec51ee..2d648723c 100644 --- a/docs/user-guide/quickstart.md +++ b/docs/user-guide/quickstart.md @@ -1,11 +1,9 @@ # VirtualShip Quickstart Guide 🚢 ```{warning} -This quickstart guide is currently out of date with the latest version of VirtualShip (v1.0.0). +This quickstart guide is designed for use with VirtualShip v0.2.2 and currently out of date with the latest version of VirtualShip (v1.0.0). It will be updated soon. -It will be updated soon. - -In particular, the `virtualship fetch` command is no longer supported. Instead, data fetching is now integrated into the `virtualship run` command. See [#226](https://github.com/Parcels-code/virtualship/pull/226) for more details in the meantime. +In particular, the `virtualship fetch` command is no longer supported. Instead, data fetching is now integrated into the `virtualship run` command. See [#226](https://github.com/Parcels-code/virtualship/pull/226) for details in the meantime. ``` Welcome to this Quickstart to using VirtualShip. In this guide we will conduct a virtual expedition in the North Sea. Note, however, that you can plan your own expedition anywhere in the global ocean and conduct whatever set of measurements you wish! From 554f2814eca220a67ee25ff4771ed25c386da917 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 13 Nov 2025 22:00:25 +0100 Subject: [PATCH 79/97] add more checks to docs compliance testing --- tests/test_utils.py | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 87e402bbf..ba13447ac 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -218,13 +218,18 @@ def test_find_nc_file_with_variable_substring(tmp_path): assert found_var == "uo_glor" -def test_data_dir_structure_compliance(): +def test_data_dir_and_filename_compliance(): """ - Test that Instrument._generate_fieldset and _get_bathy_data use the expected subdirectory names. + Test compliance of data directory structure and filename patterns as sought by base.py methods relative to as is described in the docs. + + Test that: + - Instrument._generate_fieldset and _get_bathy_data use the expected subdirectory names. + - The expected filename date pattern (YYYY_MM_DD) is used in _find_files_in_timerange. + ('phys', 'bgc', 'bathymetry') for local data loading, as required by documentation. - To avoid drift from what expecations are laid out in the docs. + To avoid drift between code implementation and what expectations are laid out in the docs. """ base_path = Path(__file__).parent.parent / "src/virtualship/instruments/base.py" utils_path = Path(__file__).parent.parent / "src/virtualship/utils.py" @@ -234,13 +239,26 @@ def test_data_dir_structure_compliance(): # Check for phys and bgc in Instrument._generate_fieldset assert 'self.from_data.joinpath("phys")' in base_code, ( - "Expected 'phys' subdirectory not found in Instrument._generate_fieldset." + "Expected 'phys' subdirectory not found in Instrument._generate_fieldset. This indicates a drift between docs and implementation." ) assert 'self.from_data.joinpath("bgc")' in base_code, ( - "Expected 'bgc' subdirectory not found in Instrument._generate_fieldset." + "Expected 'bgc' subdirectory not found in Instrument._generate_fieldset. This indicates a drift between docs and implementation." ) # Check for bathymetry in _get_bathy_data assert 'from_data.joinpath("bathymetry")' in utils_code, ( - "Expected 'bathymetry' subdirectory not found in _get_bathy_data." + "Expected 'bathymetry' subdirectory not found in _get_bathy_data. This indicates a drift between docs and implementation." + ) + + # Check for date_pattern in _find_files_in_timerange + assert 'date_pattern=r"\\d{4}_\\d{2}_\\d{2}"' in base_code, ( + "Expected date_pattern r'\\d{4}_\\d{2}_\\d{2}' not found in _find_files_in_timerange. This indicates a drift between docs and implementation." + ) + + # Check for P1D and P1M in t_resolution logic + assert 'if all("P1D" in s for s in all_files):' in base_code, ( + "Expected check for 'P1D' in all_files not found in _find_files_in_timerange. This indicates a drift between docs and implementation." + ) + assert 'elif all("P1M" in s for s in all_files):' in base_code, ( + "Expected check for 'P1M' in all_files not found in _find_files_in_timerange. This indicates a drift between docs and implementation." ) From 48937311ab840b5e855c3d67ef46f3608e7ac117 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Fri, 14 Nov 2025 08:48:26 +0100 Subject: [PATCH 80/97] TODO in readme --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index b9a59e706..c6ed488c1 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,8 @@ + + VirtualShipParcels is a command line simulator allowing students to plan and conduct a virtual research expedition, receiving measurements as if they were coming from actual oceanographic instruments including: From 5a6154167bc42179898e2148fcf0e264c0d76e88 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 14 Nov 2025 07:49:02 +0000 Subject: [PATCH 81/97] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/virtualship/instruments/adcp.py | 2 +- src/virtualship/instruments/argo_float.py | 2 +- src/virtualship/instruments/base.py | 2 +- src/virtualship/instruments/ctd.py | 2 +- src/virtualship/instruments/ctd_bgc.py | 2 +- src/virtualship/instruments/drifter.py | 2 +- src/virtualship/instruments/ship_underwater_st.py | 2 +- src/virtualship/instruments/xbt.py | 2 +- src/virtualship/utils.py | 2 +- tests/expedition/test_expedition.py | 2 +- tests/instruments/test_adcp.py | 2 +- tests/instruments/test_argo_float.py | 2 +- tests/instruments/test_ctd.py | 2 +- tests/instruments/test_ctd_bgc.py | 2 +- tests/instruments/test_drifter.py | 2 +- tests/instruments/test_ship_underwater_st.py | 2 +- tests/instruments/test_xbt.py | 2 +- tests/test_utils.py | 2 +- 18 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index 8dca31a3e..c53dbc1cf 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -2,8 +2,8 @@ from typing import ClassVar import numpy as np - from parcels import ParticleSet, ScipyParticle, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.utils import ( diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index 8ba794ba2..098774712 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -4,7 +4,6 @@ from typing import ClassVar import numpy as np - from parcels import ( AdvectionRK4, JITParticle, @@ -12,6 +11,7 @@ StatusCode, Variable, ) + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 54b45822a..b5b46abdd 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -8,9 +8,9 @@ import copernicusmarine import xarray as xr +from parcels import FieldSet from yaspin import yaspin -from parcels import FieldSet from virtualship.errors import CopernicusCatalogueError from virtualship.utils import ( COPERNICUSMARINE_PHYS_VARIABLES, diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 04fb24edd..4684c822f 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -3,8 +3,8 @@ from typing import TYPE_CHECKING, ClassVar import numpy as np - from parcels import JITParticle, ParticleSet, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 59dc4bc23..d3adcec3f 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np - from parcels import JITParticle, ParticleSet, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index 8ad0fe197..3ee8ad1ca 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np - from parcels import AdvectionRK4, JITParticle, ParticleSet, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index a28ae12db..6d8d45753 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -2,8 +2,8 @@ from typing import ClassVar import numpy as np - from parcels import ParticleSet, ScipyParticle, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.utils import add_dummy_UV, register_instrument diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index 5972efffe..df68122cf 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np - from parcels import JITParticle, ParticleSet, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index e4a134035..84839b766 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -11,8 +11,8 @@ import copernicusmarine import numpy as np import xarray as xr - from parcels import FieldSet + from virtualship.errors import CopernicusCatalogueError if TYPE_CHECKING: diff --git a/tests/expedition/test_expedition.py b/tests/expedition/test_expedition.py index 9e08ecfce..78fff2c22 100644 --- a/tests/expedition/test_expedition.py +++ b/tests/expedition/test_expedition.py @@ -6,8 +6,8 @@ import pyproj import pytest import xarray as xr - from parcels import FieldSet + from virtualship.errors import InstrumentsConfigError, ScheduleError from virtualship.models import ( Expedition, diff --git a/tests/instruments/test_adcp.py b/tests/instruments/test_adcp.py index 0f9003d69..f7802dc7e 100644 --- a/tests/instruments/test_adcp.py +++ b/tests/instruments/test_adcp.py @@ -4,8 +4,8 @@ import numpy as np import xarray as xr - from parcels import FieldSet + from virtualship.instruments.adcp import ADCPInstrument from virtualship.models import Location, Spacetime diff --git a/tests/instruments/test_argo_float.py b/tests/instruments/test_argo_float.py index 045a7b7b9..f974e19b5 100644 --- a/tests/instruments/test_argo_float.py +++ b/tests/instruments/test_argo_float.py @@ -4,8 +4,8 @@ import numpy as np import xarray as xr - from parcels import FieldSet + from virtualship.instruments.argo_float import ArgoFloat, ArgoFloatInstrument from virtualship.models import Location, Spacetime diff --git a/tests/instruments/test_ctd.py b/tests/instruments/test_ctd.py index 39b6cf47b..cde67a903 100644 --- a/tests/instruments/test_ctd.py +++ b/tests/instruments/test_ctd.py @@ -8,8 +8,8 @@ import numpy as np import xarray as xr - from parcels import Field, FieldSet + from virtualship.instruments.ctd import CTD, CTDInstrument from virtualship.models import Location, Spacetime diff --git a/tests/instruments/test_ctd_bgc.py b/tests/instruments/test_ctd_bgc.py index 4495f3e06..c58365422 100644 --- a/tests/instruments/test_ctd_bgc.py +++ b/tests/instruments/test_ctd_bgc.py @@ -8,8 +8,8 @@ import numpy as np import xarray as xr - from parcels import Field, FieldSet + from virtualship.instruments.ctd_bgc import CTD_BGC, CTD_BGCInstrument from virtualship.models import Location, Spacetime diff --git a/tests/instruments/test_drifter.py b/tests/instruments/test_drifter.py index 095e6cdfe..d160a09f5 100644 --- a/tests/instruments/test_drifter.py +++ b/tests/instruments/test_drifter.py @@ -4,8 +4,8 @@ import numpy as np import xarray as xr - from parcels import FieldSet + from virtualship.instruments.drifter import Drifter, DrifterInstrument from virtualship.models import Location, Spacetime diff --git a/tests/instruments/test_ship_underwater_st.py b/tests/instruments/test_ship_underwater_st.py index 8e1cfbdc0..e5acdd0e3 100644 --- a/tests/instruments/test_ship_underwater_st.py +++ b/tests/instruments/test_ship_underwater_st.py @@ -4,8 +4,8 @@ import numpy as np import xarray as xr - from parcels import FieldSet + from virtualship.instruments.ship_underwater_st import Underwater_STInstrument from virtualship.models import Location, Spacetime diff --git a/tests/instruments/test_xbt.py b/tests/instruments/test_xbt.py index 7bafef9f4..6c33a6dc2 100644 --- a/tests/instruments/test_xbt.py +++ b/tests/instruments/test_xbt.py @@ -8,8 +8,8 @@ import numpy as np import xarray as xr - from parcels import Field, FieldSet + from virtualship.instruments.xbt import XBT, XBTInstrument from virtualship.models import Location, Spacetime diff --git a/tests/test_utils.py b/tests/test_utils.py index ba13447ac..3018411b8 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -3,9 +3,9 @@ import numpy as np import pytest import xarray as xr +from parcels import FieldSet import virtualship.utils -from parcels import FieldSet from virtualship.models.expedition import Expedition from virtualship.utils import ( _find_nc_file_with_variable, From 00c4455dde279cd07b1c5ce883763b5b14a5c828 Mon Sep 17 00:00:00 2001 From: Jamie Atkins <106238905+j-atkins@users.noreply.github.com> Date: Fri, 14 Nov 2025 15:29:02 +0100 Subject: [PATCH 82/97] Update docs/user-guide/documentation/pre_download_data.md Co-authored-by: Erik van Sebille --- docs/user-guide/documentation/pre_download_data.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user-guide/documentation/pre_download_data.md b/docs/user-guide/documentation/pre_download_data.md index f8d7e46aa..4e49b21c1 100644 --- a/docs/user-guide/documentation/pre_download_data.md +++ b/docs/user-guide/documentation/pre_download_data.md @@ -12,7 +12,7 @@ See the [for example...](#for-example) section for an example data download work ### Data requirements -When using pre-downloaded data with VirtualShip, the software supports only: daily and monthly resolution physical and biogeochemical data, along with a static bathymetry file. +For pre-downloaded data, VirtualShip only supports daily and monthly resolution physical and biogeochemical data, along with a static bathymetry file. In addition, all pre-downloaded data must be split into separate files per timestep (i.e. one .nc file per day or month). From 3cdfd1a4c5420c88fa19a827e6c38d0b4bf531d6 Mon Sep 17 00:00:00 2001 From: Jamie Atkins <106238905+j-atkins@users.noreply.github.com> Date: Fri, 14 Nov 2025 15:32:07 +0100 Subject: [PATCH 83/97] Apply suggestions from code review Co-authored-by: Erik van Sebille --- docs/user-guide/documentation/pre_download_data.md | 4 ++-- src/virtualship/instruments/base.py | 2 +- src/virtualship/utils.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/user-guide/documentation/pre_download_data.md b/docs/user-guide/documentation/pre_download_data.md index 4e49b21c1..fc55f3393 100644 --- a/docs/user-guide/documentation/pre_download_data.md +++ b/docs/user-guide/documentation/pre_download_data.md @@ -48,7 +48,7 @@ Within these subdirectories, the expected filename conventions are: - `cmems_mod_glo_phy_anfc_0.083deg_static_bathymetry.nc` ```{tip} -Careful to use an underscore (`_`) as the separator between date components in the filenames (i.e. `YYYY_MM_DD`). +Take care to use an underscore (`_`) as the separator between date components in the filenames (i.e. `YYYY_MM_DD`). ``` ```{note} @@ -76,7 +76,7 @@ The following assumptions are also made about the data: #### Also of note -1. Whilst not mandatory to use data downloaded only from Copernicus Marine (any existing data you may hold can be re-organised accordingly), the assumptions made by VirtualShip regarding directory structure and filename conventions are motivated by alignment with the Copernicus Marine's practices. +1. Whilst not mandatory to use data downloaded only from the Copernicus Marine Service (any existing data you may hold can be re-organised accordingly), the assumptions made by VirtualShip regarding directory structure and filename conventions are motivated by alignment with the Copernicus Marine Service's practices. - If you want to use pre-existing data with VirtualShip, which you may have accessed from a different source, it is possible to do so by restructuring and/or renaming your data files as necessary. 2. The whole VirtualShip pre-downloaded data workflow should support global data or subsets thereof, provided the data files contain the necessary variables and are structured as outlined above. diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index b5b46abdd..14a2179a1 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -198,7 +198,7 @@ def _generate_fieldset(self) -> FieldSet: dimensions=self.dimensions, mesh="spherical", ) - else: # steam via Copernicus Marine + else: # stream via Copernicus Marine Service physical = var in COPERNICUSMARINE_PHYS_VARIABLES ds = self._get_copernicus_ds(physical=physical, var=var) fs = FieldSet.from_xarray_dataset( diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index 84839b766..a8f61b497 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -434,7 +434,7 @@ def _get_bathy_data( ds_bathymetry, bathymetry_variables, bathymetry_dimensions ) - else: # stream via Copernicus Marine + else: # stream via Copernicus Marine Service ds_bathymetry = copernicusmarine.open_dataset( dataset_id="cmems_mod_glo_phy_my_0.083deg_static", minimum_longitude=space_time_region.spatial_range.minimum_longitude From a84c8fd479e424be4a0583c364e9e2584df3d9b9 Mon Sep 17 00:00:00 2001 From: Jamie Atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 18 Nov 2025 13:26:29 +0100 Subject: [PATCH 84/97] Set t_min to first day of month for monthly resolution Adjust t_min to the first day of the month based on schedule start date. --- src/virtualship/instruments/base.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 14a2179a1..282babe80 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -245,7 +245,9 @@ def _find_files_in_timerange( ) if t_resolution == "monthly": - t_min = schedule_start.date() + t_min = schedule_start.date().replace( + day=1 + ) # first day of month of the schedule start date t_max = ( schedule_end.date() + timedelta( From 4198406acf0de6ee2ceed1ead80457376758e350 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 20 Nov 2025 09:33:26 +0100 Subject: [PATCH 85/97] remove redundant parameters from instrument classes --- src/virtualship/cli/_run.py | 1 - src/virtualship/instruments/adcp.py | 11 ++--------- src/virtualship/instruments/argo_float.py | 13 ++----------- src/virtualship/instruments/base.py | 18 +++++++----------- src/virtualship/instruments/ctd.py | 11 ++--------- src/virtualship/instruments/ctd_bgc.py | 16 ++-------------- src/virtualship/instruments/drifter.py | 12 ++---------- .../instruments/ship_underwater_st.py | 11 ++--------- src/virtualship/instruments/xbt.py | 13 ++----------- tests/cli/test_run.py | 3 +-- tests/instruments/test_adcp.py | 6 +++--- tests/instruments/test_argo_float.py | 5 ++--- tests/instruments/test_base.py | 15 --------------- tests/instruments/test_ctd.py | 6 +++--- tests/instruments/test_ctd_bgc.py | 5 ++--- tests/instruments/test_drifter.py | 6 +++--- tests/instruments/test_ship_underwater_st.py | 6 +++--- tests/instruments/test_xbt.py | 6 +++--- 18 files changed, 41 insertions(+), 123 deletions(-) diff --git a/src/virtualship/cli/_run.py b/src/virtualship/cli/_run.py index 06337cf3e..6ae7a1cd1 100644 --- a/src/virtualship/cli/_run.py +++ b/src/virtualship/cli/_run.py @@ -142,7 +142,6 @@ def _run(expedition_dir: str | Path, from_data: Path | None = None) -> None: # initialise instrument instrument = instrument_class( expedition=expedition, - directory=expedition_dir, from_data=Path(from_data) if from_data is not None else None, ) diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index c53dbc1cf..46f34faf8 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -2,8 +2,8 @@ from typing import ClassVar import numpy as np -from parcels import ParticleSet, ScipyParticle, Variable +from parcels import ParticleSet, ScipyParticle, Variable from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.utils import ( @@ -54,19 +54,12 @@ def _sample_velocity(particle, fieldset, time): class ADCPInstrument(Instrument): """ADCP instrument class.""" - def __init__(self, expedition, directory, from_data): + def __init__(self, expedition, from_data): """Initialize ADCPInstrument.""" - filenames = { - "U": f"{ADCP.name}_uv.nc", - "V": f"{ADCP.name}_uv.nc", - } variables = {"U": "uo", "V": "vo"} super().__init__( - ADCP.name, expedition, - directory, - filenames, variables, add_bathymetry=False, allow_time_extrapolation=True, diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index 098774712..69095da91 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -4,6 +4,7 @@ from typing import ClassVar import numpy as np + from parcels import ( AdvectionRK4, JITParticle, @@ -11,7 +12,6 @@ StatusCode, Variable, ) - from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime @@ -137,14 +137,8 @@ def _check_error(particle, fieldset, time): class ArgoFloatInstrument(Instrument): """ArgoFloat instrument class.""" - def __init__(self, expedition, directory, from_data): + def __init__(self, expedition, from_data): """Initialize ArgoFloatInstrument.""" - filenames = { - "U": f"{ArgoFloat.name}_uv.nc", - "V": f"{ArgoFloat.name}_uv.nc", - "S": f"{ArgoFloat.name}_s.nc", - "T": f"{ArgoFloat.name}_t.nc", - } variables = {"U": "uo", "V": "vo", "S": "so", "T": "thetao"} buffer_spec = { "latlon": 3.0, # [degrees] @@ -152,10 +146,7 @@ def __init__(self, expedition, directory, from_data): } super().__init__( - ArgoFloat.name, expedition, - directory, - filenames, variables, add_bathymetry=False, allow_time_extrapolation=False, diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 282babe80..d9339942f 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import abc import glob import re @@ -8,9 +10,9 @@ import copernicusmarine import xarray as xr -from parcels import FieldSet from yaspin import yaspin +from parcels import FieldSet from virtualship.errors import CopernicusCatalogueError from virtualship.utils import ( COPERNICUSMARINE_PHYS_VARIABLES, @@ -29,10 +31,7 @@ class Instrument(abc.ABC): def __init__( self, - name: str, - expedition: "Expedition", - directory: Path | str, - filenames: dict, + expedition: Expedition, variables: dict, add_bathymetry: bool, allow_time_extrapolation: bool, @@ -42,10 +41,7 @@ def __init__( limit_spec: dict | None = None, ): """Initialise instrument.""" - self.name = name self.expedition = expedition - self.directory = directory - self.filenames = filenames self.from_data = from_data self.variables = OrderedDict(variables) @@ -67,7 +63,7 @@ def load_input_data(self) -> FieldSet: fieldset = self._generate_fieldset() except Exception as e: raise CopernicusCatalogueError( - f"Failed to load input data directly from Copernicus Marine (or local data) for instrument '{self.name}'. Original error: {e}" + f"Failed to load input data directly from Copernicus Marine (or local data) for instrument '{self.__class__.__name__}'. Original error: {e}" ) from e # interpolation methods @@ -105,14 +101,14 @@ def execute(self, measurements: list, out_path: str | Path) -> None: """Run instrument simulation.""" if not self.verbose_progress: with yaspin( - text=f"Simulating {self.name} measurements... ", + text=f"Simulating {self.__class__.__name__.split('Instrument')[0]} measurements... ", side="right", spinner=ship_spinner, ) as spinner: self.simulate(measurements, out_path) spinner.ok("✅\n") else: - print(f"Simulating {self.name} measurements... ") + print(f"Simulating {self.__class__.__name__} measurements... ") self.simulate(measurements, out_path) print("\n") diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 4684c822f..262fc7999 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -3,8 +3,8 @@ from typing import TYPE_CHECKING, ClassVar import numpy as np -from parcels import JITParticle, ParticleSet, Variable +from parcels import JITParticle, ParticleSet, Variable from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType @@ -79,19 +79,12 @@ def _ctd_cast(particle, fieldset, time): class CTDInstrument(Instrument): """CTD instrument class.""" - def __init__(self, expedition, directory, from_data): + def __init__(self, expedition, from_data): """Initialize CTDInstrument.""" - filenames = { - "S": f"{CTD.name}_s.nc", - "T": f"{CTD.name}_t.nc", - } variables = {"S": "so", "T": "thetao"} super().__init__( - CTD.name, expedition, - directory, - filenames, variables, add_bathymetry=True, allow_time_extrapolation=True, diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index d3adcec3f..1a9ba6127 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np -from parcels import JITParticle, ParticleSet, Variable +from parcels import JITParticle, ParticleSet, Variable from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime @@ -101,17 +101,8 @@ def _ctd_bgc_cast(particle, fieldset, time): class CTD_BGCInstrument(Instrument): """CTD_BGC instrument class.""" - def __init__(self, expedition, directory, from_data): + def __init__(self, expedition, from_data): """Initialize CTD_BGCInstrument.""" - filenames = { - "o2": f"{CTD_BGC.name}_o2.nc", - "chl": f"{CTD_BGC.name}_chl.nc", - "no3": f"{CTD_BGC.name}_no3.nc", - "po4": f"{CTD_BGC.name}_po4.nc", - "ph": f"{CTD_BGC.name}_ph.nc", - "phyc": f"{CTD_BGC.name}_phyc.nc", - "nppv": f"{CTD_BGC.name}_nppv.nc", - } variables = { "o2": "o2", "chl": "chl", @@ -122,10 +113,7 @@ def __init__(self, expedition, directory, from_data): "nppv": "nppv", } super().__init__( - CTD_BGC.name, expedition, - directory, - filenames, variables, add_bathymetry=True, allow_time_extrapolation=True, diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index 3ee8ad1ca..ddc877b69 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np -from parcels import AdvectionRK4, JITParticle, ParticleSet, Variable +from parcels import AdvectionRK4, JITParticle, ParticleSet, Variable from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime @@ -63,13 +63,8 @@ def _check_lifetime(particle, fieldset, time): class DrifterInstrument(Instrument): """Drifter instrument class.""" - def __init__(self, expedition, directory, from_data): + def __init__(self, expedition, from_data): """Initialize DrifterInstrument.""" - filenames = { - "U": f"{Drifter.name}_uv.nc", - "V": f"{Drifter.name}_uv.nc", - "T": f"{Drifter.name}_t.nc", - } variables = {"U": "uo", "V": "vo", "T": "thetao"} buffer_spec = { "latlon": 6.0, # [degrees] @@ -81,10 +76,7 @@ def __init__(self, expedition, directory, from_data): } super().__init__( - Drifter.name, expedition, - directory, - filenames, variables, add_bathymetry=False, allow_time_extrapolation=False, diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index 6d8d45753..52d2976b7 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -2,8 +2,8 @@ from typing import ClassVar import numpy as np -from parcels import ParticleSet, ScipyParticle, Variable +from parcels import ParticleSet, ScipyParticle, Variable from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.utils import add_dummy_UV, register_instrument @@ -55,19 +55,12 @@ def _sample_temperature(particle, fieldset, time): class Underwater_STInstrument(Instrument): """Underwater_ST instrument class.""" - def __init__(self, expedition, directory, from_data): + def __init__(self, expedition, from_data): """Initialize Underwater_STInstrument.""" - filenames = { - "S": f"{Underwater_ST.name}_s.nc", - "T": f"{Underwater_ST.name}_t.nc", - } variables = {"S": "so", "T": "thetao"} super().__init__( - Underwater_ST.name, expedition, - directory, - filenames, variables, add_bathymetry=False, allow_time_extrapolation=True, diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index df68122cf..ef1339c5d 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np -from parcels import JITParticle, ParticleSet, Variable +from parcels import JITParticle, ParticleSet, Variable from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime @@ -77,20 +77,11 @@ def _xbt_cast(particle, fieldset, time): class XBTInstrument(Instrument): """XBT instrument class.""" - def __init__(self, expedition, directory, from_data): + def __init__(self, expedition, from_data): """Initialize XBTInstrument.""" - filenames = { - "U": f"{XBT.name}_uv.nc", - "V": f"{XBT.name}_uv.nc", - "S": f"{XBT.name}_s.nc", - "T": f"{XBT.name}_t.nc", - } variables = {"U": "uo", "V": "vo", "S": "so", "T": "thetao"} super().__init__( - XBT.name, expedition, - directory, - filenames, variables, add_bathymetry=True, allow_time_extrapolation=True, diff --git a/tests/cli/test_run.py b/tests/cli/test_run.py index daa822f01..5ad14c648 100644 --- a/tests/cli/test_run.py +++ b/tests/cli/test_run.py @@ -19,10 +19,9 @@ def _simulate_schedule(projection, expedition): class DummyInstrument: """Dummy instrument class that just creates empty output directories.""" - def __init__(self, expedition, directory, from_data=None): + def __init__(self, expedition, from_data=None): """Initialize DummyInstrument.""" self.expedition = expedition - self.directory = Path(directory) self.from_data = from_data def execute(self, measurements, out_path): diff --git a/tests/instruments/test_adcp.py b/tests/instruments/test_adcp.py index f7802dc7e..d87dae423 100644 --- a/tests/instruments/test_adcp.py +++ b/tests/instruments/test_adcp.py @@ -4,8 +4,8 @@ import numpy as np import xarray as xr -from parcels import FieldSet +from parcels import FieldSet from virtualship.instruments.adcp import ADCPInstrument from virtualship.models import Location, Spacetime @@ -85,10 +85,10 @@ class adcp_config: num_bins = NUM_BINS expedition = DummyExpedition() - directory = tmpdir + from_data = None - adcp_instrument = ADCPInstrument(expedition, directory, from_data) + adcp_instrument = ADCPInstrument(expedition, from_data) out_path = tmpdir.join("out.zarr") adcp_instrument.load_input_data = lambda: fieldset diff --git a/tests/instruments/test_argo_float.py b/tests/instruments/test_argo_float.py index f974e19b5..1dfa4060b 100644 --- a/tests/instruments/test_argo_float.py +++ b/tests/instruments/test_argo_float.py @@ -4,8 +4,8 @@ import numpy as np import xarray as xr -from parcels import FieldSet +from parcels import FieldSet from virtualship.instruments.argo_float import ArgoFloat, ArgoFloatInstrument from virtualship.models import Location, Spacetime @@ -58,10 +58,9 @@ class DummyExpedition: pass expedition = DummyExpedition() - directory = tmpdir from_data = None - argo_instrument = ArgoFloatInstrument(expedition, directory, from_data) + argo_instrument = ArgoFloatInstrument(expedition, from_data) out_path = tmpdir.join("out.zarr") argo_instrument.load_input_data = lambda: fieldset diff --git a/tests/instruments/test_base.py b/tests/instruments/test_base.py index 272290e67..6aec1b663 100644 --- a/tests/instruments/test_base.py +++ b/tests/instruments/test_base.py @@ -33,10 +33,7 @@ def test_load_input_data(mock_copernicusmarine, mock_select_product_id, mock_Fie mock_fieldset.__getitem__.side_effect = lambda k: MagicMock() mock_copernicusmarine.open_dataset.return_value = MagicMock() dummy = DummyInstrument( - name="test", expedition=MagicMock(schedule=MagicMock(space_time_region=MagicMock())), - directory="/tmp", - filenames={"A": "a.nc"}, variables={"A": "a"}, add_bathymetry=False, allow_time_extrapolation=False, @@ -51,10 +48,7 @@ def test_load_input_data(mock_copernicusmarine, mock_select_product_id, mock_Fie def test_execute_calls_simulate(monkeypatch): dummy = DummyInstrument( - name="test", expedition=MagicMock(schedule=MagicMock(space_time_region=MagicMock())), - directory="/tmp", - filenames={"A": "a.nc"}, variables={"A": "a"}, add_bathymetry=False, allow_time_extrapolation=False, @@ -68,10 +62,7 @@ def test_execute_calls_simulate(monkeypatch): def test_get_spec_value_buffer_and_limit(): dummy = DummyInstrument( - name="test", expedition=MagicMock(schedule=MagicMock(space_time_region=MagicMock())), - directory="/tmp", - filenames={"A": "a.nc"}, variables={"A": "a"}, add_bathymetry=False, allow_time_extrapolation=False, @@ -87,10 +78,7 @@ def test_get_spec_value_buffer_and_limit(): def test_generate_fieldset_combines_fields(monkeypatch): dummy = DummyInstrument( - name="test", expedition=MagicMock(schedule=MagicMock(space_time_region=MagicMock())), - directory="/tmp", - filenames={"A": "a.nc", "B": "b.nc"}, variables={"A": "a", "B": "b"}, add_bathymetry=False, allow_time_extrapolation=False, @@ -115,10 +103,7 @@ def test_generate_fieldset_combines_fields(monkeypatch): def test_load_input_data_error(monkeypatch): dummy = DummyInstrument( - name="test", expedition=MagicMock(schedule=MagicMock(space_time_region=MagicMock())), - directory="/tmp", - filenames={"A": "a.nc"}, variables={"A": "a"}, add_bathymetry=False, allow_time_extrapolation=False, diff --git a/tests/instruments/test_ctd.py b/tests/instruments/test_ctd.py index cde67a903..0382e1597 100644 --- a/tests/instruments/test_ctd.py +++ b/tests/instruments/test_ctd.py @@ -8,8 +8,8 @@ import numpy as np import xarray as xr -from parcels import Field, FieldSet +from parcels import Field, FieldSet from virtualship.instruments.ctd import CTD, CTDInstrument from virtualship.models import Location, Spacetime @@ -129,10 +129,10 @@ class space_time_region: )() expedition = DummyExpedition() - directory = tmpdir + from_data = None - ctd_instrument = CTDInstrument(expedition, directory, from_data) + ctd_instrument = CTDInstrument(expedition, from_data) out_path = tmpdir.join("out.zarr") ctd_instrument.load_input_data = lambda: fieldset diff --git a/tests/instruments/test_ctd_bgc.py b/tests/instruments/test_ctd_bgc.py index c58365422..7d1676ed8 100644 --- a/tests/instruments/test_ctd_bgc.py +++ b/tests/instruments/test_ctd_bgc.py @@ -8,8 +8,8 @@ import numpy as np import xarray as xr -from parcels import Field, FieldSet +from parcels import Field, FieldSet from virtualship.instruments.ctd_bgc import CTD_BGC, CTD_BGCInstrument from virtualship.models import Location, Spacetime @@ -167,10 +167,9 @@ class DummyExpedition: pass expedition = DummyExpedition() - directory = tmpdir from_data = None - ctd_bgc_instrument = CTD_BGCInstrument(expedition, directory, from_data) + ctd_bgc_instrument = CTD_BGCInstrument(expedition, from_data) out_path = tmpdir.join("out.zarr") ctd_bgc_instrument.load_input_data = lambda: fieldset diff --git a/tests/instruments/test_drifter.py b/tests/instruments/test_drifter.py index d160a09f5..27bc7ffdd 100644 --- a/tests/instruments/test_drifter.py +++ b/tests/instruments/test_drifter.py @@ -4,8 +4,8 @@ import numpy as np import xarray as xr -from parcels import FieldSet +from parcels import FieldSet from virtualship.instruments.drifter import Drifter, DrifterInstrument from virtualship.models import Location, Spacetime @@ -57,10 +57,10 @@ class DummyExpedition: pass expedition = DummyExpedition() - directory = tmpdir + from_data = None - drifter_instrument = DrifterInstrument(expedition, directory, from_data) + drifter_instrument = DrifterInstrument(expedition, from_data) out_path = tmpdir.join("out.zarr") drifter_instrument.load_input_data = lambda: fieldset diff --git a/tests/instruments/test_ship_underwater_st.py b/tests/instruments/test_ship_underwater_st.py index e5acdd0e3..96778f87f 100644 --- a/tests/instruments/test_ship_underwater_st.py +++ b/tests/instruments/test_ship_underwater_st.py @@ -4,8 +4,8 @@ import numpy as np import xarray as xr -from parcels import FieldSet +from parcels import FieldSet from virtualship.instruments.ship_underwater_st import Underwater_STInstrument from virtualship.models import Location, Spacetime @@ -72,10 +72,10 @@ class DummyExpedition: pass expedition = DummyExpedition() - directory = tmpdir + from_data = None - st_instrument = Underwater_STInstrument(expedition, directory, from_data) + st_instrument = Underwater_STInstrument(expedition, from_data) out_path = tmpdir.join("out.zarr") st_instrument.load_input_data = lambda: fieldset diff --git a/tests/instruments/test_xbt.py b/tests/instruments/test_xbt.py index 6c33a6dc2..524125101 100644 --- a/tests/instruments/test_xbt.py +++ b/tests/instruments/test_xbt.py @@ -8,8 +8,8 @@ import numpy as np import xarray as xr -from parcels import Field, FieldSet +from parcels import Field, FieldSet from virtualship.instruments.xbt import XBT, XBTInstrument from virtualship.models import Location, Spacetime @@ -100,10 +100,10 @@ class DummyExpedition: pass expedition = DummyExpedition() - directory = tmpdir + from_data = None - xbt_instrument = XBTInstrument(expedition, directory, from_data) + xbt_instrument = XBTInstrument(expedition, from_data) out_path = tmpdir.join("out.zarr") xbt_instrument.load_input_data = lambda: fieldset From 94d5be28f87c59428c8860ef9795411f7786dde1 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 20 Nov 2025 10:04:00 +0100 Subject: [PATCH 86/97] change variable name --- src/virtualship/instruments/adcp.py | 2 +- src/virtualship/instruments/argo_float.py | 4 ++-- src/virtualship/instruments/base.py | 12 ++++++------ src/virtualship/instruments/ctd.py | 2 +- src/virtualship/instruments/ctd_bgc.py | 2 +- src/virtualship/instruments/drifter.py | 4 ++-- src/virtualship/instruments/ship_underwater_st.py | 2 +- src/virtualship/instruments/xbt.py | 2 +- tests/instruments/test_base.py | 2 +- 9 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index 46f34faf8..0270b3e68 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -64,7 +64,7 @@ def __init__(self, expedition, from_data): add_bathymetry=False, allow_time_extrapolation=True, verbose_progress=False, - buffer_spec=None, + spacetime_buffer_size=None, limit_spec=None, from_data=from_data, ) diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index 69095da91..2253c1a12 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -140,7 +140,7 @@ class ArgoFloatInstrument(Instrument): def __init__(self, expedition, from_data): """Initialize ArgoFloatInstrument.""" variables = {"U": "uo", "V": "vo", "S": "so", "T": "thetao"} - buffer_spec = { + spacetime_buffer_size = { "latlon": 3.0, # [degrees] "time": 21.0, # [days] } @@ -151,7 +151,7 @@ def __init__(self, expedition, from_data): add_bathymetry=False, allow_time_extrapolation=False, verbose_progress=True, - buffer_spec=buffer_spec, + spacetime_buffer_size=spacetime_buffer_size, limit_spec=None, from_data=from_data, ) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index d9339942f..e837ba793 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -37,7 +37,7 @@ def __init__( allow_time_extrapolation: bool, verbose_progress: bool, from_data: Path | None, - buffer_spec: dict | None = None, + spacetime_buffer_size: dict | None = None, limit_spec: dict | None = None, ): """Initialise instrument.""" @@ -54,7 +54,7 @@ def __init__( self.add_bathymetry = add_bathymetry self.allow_time_extrapolation = allow_time_extrapolation self.verbose_progress = verbose_progress - self.buffer_spec = buffer_spec + self.spacetime_buffer_size = spacetime_buffer_size self.limit_spec = limit_spec def load_input_data(self) -> FieldSet: @@ -78,8 +78,8 @@ def load_input_data(self) -> FieldSet: if self.add_bathymetry: bathymetry_field = _get_bathy_data( self.expedition.schedule.space_time_region, - latlon_buffer=self.buffer_spec.get("latlon") - if self.buffer_spec + latlon_buffer=self.spacetime_buffer_size.get("latlon") + if self.spacetime_buffer_size else None, from_data=self.from_data, ).bathymetry @@ -209,8 +209,8 @@ def _generate_fieldset(self) -> FieldSet: return base_fieldset def _get_spec_value(self, spec_type: str, key: str, default=None): - """Helper to extract a value from buffer_spec or limit_spec.""" - spec = self.buffer_spec if spec_type == "buffer" else self.limit_spec + """Helper to extract a value from spacetime_buffer_size or limit_spec.""" + spec = self.spacetime_buffer_size if spec_type == "buffer" else self.limit_spec return spec.get(key) if spec and spec.get(key) is not None else default def _find_files_in_timerange( diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index 262fc7999..f90c5fc0c 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -89,7 +89,7 @@ def __init__(self, expedition, from_data): add_bathymetry=True, allow_time_extrapolation=True, verbose_progress=False, - buffer_spec=None, + spacetime_buffer_size=None, limit_spec=None, from_data=from_data, ) diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 1a9ba6127..039e850c2 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -118,7 +118,7 @@ def __init__(self, expedition, from_data): add_bathymetry=True, allow_time_extrapolation=True, verbose_progress=False, - buffer_spec=None, + spacetime_buffer_size=None, limit_spec=None, from_data=from_data, ) diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index ddc877b69..8428ad7cf 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -66,7 +66,7 @@ class DrifterInstrument(Instrument): def __init__(self, expedition, from_data): """Initialize DrifterInstrument.""" variables = {"U": "uo", "V": "vo", "T": "thetao"} - buffer_spec = { + spacetime_buffer_size = { "latlon": 6.0, # [degrees] "time": 21.0, # [days] } @@ -81,7 +81,7 @@ def __init__(self, expedition, from_data): add_bathymetry=False, allow_time_extrapolation=False, verbose_progress=True, - buffer_spec=buffer_spec, + spacetime_buffer_size=spacetime_buffer_size, limit_spec=limit_spec, from_data=from_data, ) diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index 52d2976b7..e6af25b31 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -65,7 +65,7 @@ def __init__(self, expedition, from_data): add_bathymetry=False, allow_time_extrapolation=True, verbose_progress=False, - buffer_spec=None, + spacetime_buffer_size=None, limit_spec=None, from_data=from_data, ) diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index ef1339c5d..3e11b55dd 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -86,7 +86,7 @@ def __init__(self, expedition, from_data): add_bathymetry=True, allow_time_extrapolation=True, verbose_progress=False, - buffer_spec=None, + spacetime_buffer_size=None, limit_spec=None, from_data=from_data, ) diff --git a/tests/instruments/test_base.py b/tests/instruments/test_base.py index 6aec1b663..29d319ba9 100644 --- a/tests/instruments/test_base.py +++ b/tests/instruments/test_base.py @@ -67,7 +67,7 @@ def test_get_spec_value_buffer_and_limit(): add_bathymetry=False, allow_time_extrapolation=False, verbose_progress=False, - buffer_spec={"latlon": 5.0}, + spacetime_buffer_size={"latlon": 5.0}, limit_spec={"depth_min": 10.0}, from_data=None, ) From 890cd94c2cae684971bc8ab1503b9cb4ac4f8b26 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 20 Nov 2025 10:13:36 +0100 Subject: [PATCH 87/97] make _find_files_in_timerange standalone from Instrument base class --- src/virtualship/instruments/base.py | 79 +++-------------------------- src/virtualship/utils.py | 74 ++++++++++++++++++++++++++- 2 files changed, 78 insertions(+), 75 deletions(-) diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index e837ba793..5faf9a760 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -1,10 +1,8 @@ from __future__ import annotations import abc -import glob -import re from collections import OrderedDict -from datetime import datetime, timedelta +from datetime import timedelta from pathlib import Path from typing import TYPE_CHECKING @@ -16,6 +14,7 @@ from virtualship.errors import CopernicusCatalogueError from virtualship.utils import ( COPERNICUSMARINE_PHYS_VARIABLES, + _find_files_in_timerange, _find_nc_file_with_variable, _get_bathy_data, _select_product_id, @@ -108,7 +107,9 @@ def execute(self, measurements: list, out_path: str | Path) -> None: self.simulate(measurements, out_path) spinner.ok("✅\n") else: - print(f"Simulating {self.__class__.__name__} measurements... ") + print( + f"Simulating {self.__class__.__name__.split('Instrument')[0]} measurements... " + ) self.simulate(measurements, out_path) print("\n") @@ -174,7 +175,7 @@ def _generate_fieldset(self) -> FieldSet: self.expedition.schedule.space_time_region.time_range.end_time ) - files = self._find_files_in_timerange( + files = _find_files_in_timerange( data_dir, schedule_start, schedule_end, @@ -212,71 +213,3 @@ def _get_spec_value(self, spec_type: str, key: str, default=None): """Helper to extract a value from spacetime_buffer_size or limit_spec.""" spec = self.spacetime_buffer_size if spec_type == "buffer" else self.limit_spec return spec.get(key) if spec and spec.get(key) is not None else default - - def _find_files_in_timerange( - self, - data_dir: Path, - schedule_start, - schedule_end, - date_pattern=r"\d{4}_\d{2}_\d{2}", - date_fmt="%Y_%m_%d", - ) -> list: - """Find all files in data_dir whose filenames contain a date within [schedule_start, schedule_end] (inclusive).""" - # TODO: scope to make this more flexible for different date patterns / formats ... ? - - all_files = glob.glob(str(data_dir.joinpath("*"))) - if not all_files: - raise ValueError( - f"No files found in data directory {data_dir}. Please ensure the directory contains files with 'P1D' or 'P1M' in their names as per Copernicus Marine Product ID naming conventions." - ) - - if all("P1D" in s for s in all_files): - t_resolution = "daily" - elif all("P1M" in s for s in all_files): - t_resolution = "monthly" - else: - raise ValueError( - f"Could not determine time resolution from filenames in data directory. Please ensure all filenames in {data_dir} contain either 'P1D' (daily) or 'P1M' (monthly), " - f"as per the Copernicus Marine Product ID naming conventions." - ) - - if t_resolution == "monthly": - t_min = schedule_start.date().replace( - day=1 - ) # first day of month of the schedule start date - t_max = ( - schedule_end.date() - + timedelta( - days=32 - ) # buffer to ensure fieldset end date is always longer than schedule end date for monthly data - ) - else: # daily - t_min = schedule_start.date() - t_max = schedule_end.date() - - files_with_dates = [] - for file in data_dir.iterdir(): - if file.is_file(): - match = re.search(date_pattern, file.name) - if match: - file_date = datetime.strptime( - match.group(), date_fmt - ).date() # normalise to date only for comparison (given start/end dates have hour/minute components which may exceed those in file_date) - if t_min <= file_date <= t_max: - files_with_dates.append((file_date, file.name)) - - files_with_dates.sort( - key=lambda x: x[0] - ) # sort by extracted date; more robust than relying on filesystem order - - # catch if not enough data coverage found for the requested time range - if files_with_dates[-1][0] < schedule_end.date(): - raise ValueError( - f"Not enough data coverage found in {data_dir} for the requested time range {schedule_start} to {schedule_end}. " - f"Latest available data is for date {files_with_dates[-1][0]}." - f"If using monthly data, please ensure that the last month downloaded covers the schedule end date + 1 month." - f"See documentation for more details: <>" - # TODO: add link to relevant documentation! - ) - - return [fname for _, fname in files_with_dates] diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index a8f61b497..ce86b1de2 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -1,8 +1,10 @@ from __future__ import annotations +import glob import os +import re import warnings -from datetime import timedelta +from datetime import datetime, timedelta from functools import lru_cache from importlib.resources import files from pathlib import Path @@ -11,8 +13,8 @@ import copernicusmarine import numpy as np import xarray as xr -from parcels import FieldSet +from parcels import FieldSet from virtualship.errors import CopernicusCatalogueError if TYPE_CHECKING: @@ -491,3 +493,71 @@ def _find_nc_file_with_variable(data_dir: Path, var: str) -> str | None: except Exception: continue return None + + +def _find_files_in_timerange( + data_dir: Path, + schedule_start, + schedule_end, + date_pattern=r"\d{4}_\d{2}_\d{2}", + date_fmt="%Y_%m_%d", +) -> list: + """Find all files in data_dir whose filenames contain a date within [schedule_start, schedule_end] (inclusive).""" + # TODO: scope to make this more flexible for different date patterns / formats ... ? + + all_files = glob.glob(str(data_dir.joinpath("*"))) + if not all_files: + raise ValueError( + f"No files found in data directory {data_dir}. Please ensure the directory contains files with 'P1D' or 'P1M' in their names as per Copernicus Marine Product ID naming conventions." + ) + + if all("P1D" in s for s in all_files): + t_resolution = "daily" + elif all("P1M" in s for s in all_files): + t_resolution = "monthly" + else: + raise ValueError( + f"Could not determine time resolution from filenames in data directory. Please ensure all filenames in {data_dir} contain either 'P1D' (daily) or 'P1M' (monthly), " + f"as per the Copernicus Marine Product ID naming conventions." + ) + + if t_resolution == "monthly": + t_min = schedule_start.date().replace( + day=1 + ) # first day of month of the schedule start date + t_max = ( + schedule_end.date() + + timedelta( + days=32 + ) # buffer to ensure fieldset end date is always longer than schedule end date for monthly data + ) + else: # daily + t_min = schedule_start.date() + t_max = schedule_end.date() + + files_with_dates = [] + for file in data_dir.iterdir(): + if file.is_file(): + match = re.search(date_pattern, file.name) + if match: + file_date = datetime.strptime( + match.group(), date_fmt + ).date() # normalise to date only for comparison (given start/end dates have hour/minute components which may exceed those in file_date) + if t_min <= file_date <= t_max: + files_with_dates.append((file_date, file.name)) + + files_with_dates.sort( + key=lambda x: x[0] + ) # sort by extracted date; more robust than relying on filesystem order + + # catch if not enough data coverage found for the requested time range + if files_with_dates[-1][0] < schedule_end.date(): + raise ValueError( + f"Not enough data coverage found in {data_dir} for the requested time range {schedule_start} to {schedule_end}. " + f"Latest available data is for date {files_with_dates[-1][0]}." + f"If using monthly data, please ensure that the last month downloaded covers the schedule end date + 1 month." + f"See documentation for more details: <>" + # TODO: add link to relevant documentation! + ) + + return [fname for _, fname in files_with_dates] From c0ba080378031c831a688c442bebfa078f84a266 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 20 Nov 2025 10:21:25 +0100 Subject: [PATCH 88/97] Update error docstring --- src/virtualship/errors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/virtualship/errors.py b/src/virtualship/errors.py index fad5863db..ac1aa8a1b 100644 --- a/src/virtualship/errors.py +++ b/src/virtualship/errors.py @@ -23,7 +23,7 @@ class ScheduleError(RuntimeError): class InstrumentsConfigError(RuntimeError): - """An error in the config.""" + """An error in the InstrumentsConfig.""" pass From 141530a72c2f115f39caefcbaf3fda4dcd40ad88 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 20 Nov 2025 11:28:48 +0100 Subject: [PATCH 89/97] update plan UI logic to update space-time region dynamically --- src/virtualship/cli/_plan.py | 61 ++++++++++++++++++++++++++++++------ src/virtualship/utils.py | 5 ++- tests/test_utils.py | 8 ++--- 3 files changed, 58 insertions(+), 16 deletions(-) diff --git a/src/virtualship/cli/_plan.py b/src/virtualship/cli/_plan.py index a164cba31..8b2adb155 100644 --- a/src/virtualship/cli/_plan.py +++ b/src/virtualship/cli/_plan.py @@ -590,14 +590,6 @@ def _update_instrument_configs(self): ) def _update_schedule(self): - spatial_range = SpatialRange( - minimum_longitude=self.query_one("#min_lon").value, - maximum_longitude=self.query_one("#max_lon").value, - minimum_latitude=self.query_one("#min_lat").value, - maximum_latitude=self.query_one("#max_lat").value, - minimum_depth=self.query_one("#min_depth").value, - maximum_depth=self.query_one("#max_depth").value, - ) start_time_input = self.query_one("#start_time").value end_time_input = self.query_one("#end_time").value waypoint_times = [ @@ -614,8 +606,8 @@ def _update_schedule(self): else: end_time = end_time_input time_range = TimeRange(start_time=start_time, end_time=end_time) - self.expedition.schedule.space_time_region.spatial_range = spatial_range self.expedition.schedule.space_time_region.time_range = time_range + for i, wp in enumerate(self.expedition.schedule.waypoints): wp.location = Location( latitude=float(self.query_one(f"#wp{i}_lat").value), @@ -640,6 +632,57 @@ def _update_schedule(self): elif switch_on: wp.instrument.append(instrument) + # take min/max lat/lon to be most extreme values of waypoints or space_time_region inputs (so as to cover possibility of user edits in either place) + # also prevents situation where e.g. user defines a space time region inconsistent with waypoint locations and vice versa (warning also provided) + waypoint_lats = [ + wp.location.latitude for wp in self.expedition.schedule.waypoints + ] + waypoint_lons = [ + wp.location.longitude for wp in self.expedition.schedule.waypoints + ] + wp_min_lat, wp_max_lat = ( + min(waypoint_lats) if waypoint_lats else -90.0, + max(waypoint_lats) if waypoint_lats else 90.0, + ) + wp_min_lon, wp_max_lon = ( + min(waypoint_lons) if waypoint_lons else -180.0, + max(waypoint_lons) if waypoint_lons else 180.0, + ) + + st_reg_min_lat = float(self.query_one("#min_lat").value) + st_reg_max_lat = float(self.query_one("#max_lat").value) + st_reg_min_lon = float(self.query_one("#min_lon").value) + st_reg_max_lon = float(self.query_one("#max_lon").value) + + min_lat = min(wp_min_lat, st_reg_min_lat) + max_lat = max(wp_max_lat, st_reg_max_lat) + min_lon = min(wp_min_lon, st_reg_min_lon) + max_lon = max(wp_max_lon, st_reg_max_lon) + + spatial_range = SpatialRange( + minimum_longitude=min_lon, + maximum_longitude=max_lon, + minimum_latitude=min_lat, + maximum_latitude=max_lat, + minimum_depth=self.query_one("#min_depth").value, + maximum_depth=self.query_one("#max_depth").value, + ) + self.expedition.schedule.space_time_region.spatial_range = spatial_range + + # provide warning if user defines a space time region inconsistent with waypoint locations + if ( + (wp_min_lat < st_reg_min_lat) + or (wp_max_lat > st_reg_max_lat) + or (wp_min_lon < st_reg_min_lon) + or (wp_max_lon > st_reg_max_lon) + ): + self.notify( + "[b]WARNING[/b]. One or more waypoint locations lie outside the defined space-time region. Take care if manually adjusting the space-time region." + "\n\nThe space-time region will be automatically adjusted on saving to include all waypoint locations.", + severity="warning", + timeout=10, + ) + @on(Input.Changed) def show_invalid_reasons(self, event: Input.Changed) -> None: input_id = event.input.id diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index ce86b1de2..6dc25cde7 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -427,7 +427,7 @@ def _get_bathy_data( except Exception as e: # TODO: link to documentation on expected data structure!! raise RuntimeError( - f"\n\n❗️ Could not find bathymetry variable '{var}' in data directory '{from_data}/bathymetry/'.\n\n❗️ Is the pre-downloaded data directory structure compliant with VirtualShip expectations?\n\n❗️ See for more information on expectations: <<>>\n" + f"\n\n❗️ Could not find bathymetry variable '{var}' in data directory '{from_data}/bathymetry/'.\n\n❗️ Is the pre-downloaded data directory structure compliant with VirtualShip expectations?\n\n❗️ See the docs for more information on expectations: https://virtualship.readthedocs.io/en/latest/user-guide/index.html#documentation\n" ) from e ds_bathymetry = xr.open_dataset(bathy_dir.joinpath(filename)) bathymetry_variables = {"bathymetry": "deptho"} @@ -556,8 +556,7 @@ def _find_files_in_timerange( f"Not enough data coverage found in {data_dir} for the requested time range {schedule_start} to {schedule_end}. " f"Latest available data is for date {files_with_dates[-1][0]}." f"If using monthly data, please ensure that the last month downloaded covers the schedule end date + 1 month." - f"See documentation for more details: <>" - # TODO: add link to relevant documentation! + f"See the docs for more details: https://virtualship.readthedocs.io/en/latest/user-guide/index.html#documentation" ) return [fname for _, fname in files_with_dates] diff --git a/tests/test_utils.py b/tests/test_utils.py index 3018411b8..6004ea69c 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -3,9 +3,9 @@ import numpy as np import pytest import xarray as xr -from parcels import FieldSet import virtualship.utils +from parcels import FieldSet from virtualship.models.expedition import Expedition from virtualship.utils import ( _find_nc_file_with_variable, @@ -251,14 +251,14 @@ def test_data_dir_and_filename_compliance(): ) # Check for date_pattern in _find_files_in_timerange - assert 'date_pattern=r"\\d{4}_\\d{2}_\\d{2}"' in base_code, ( + assert 'date_pattern=r"\\d{4}_\\d{2}_\\d{2}"' in utils_code, ( "Expected date_pattern r'\\d{4}_\\d{2}_\\d{2}' not found in _find_files_in_timerange. This indicates a drift between docs and implementation." ) # Check for P1D and P1M in t_resolution logic - assert 'if all("P1D" in s for s in all_files):' in base_code, ( + assert 'if all("P1D" in s for s in all_files):' in utils_code, ( "Expected check for 'P1D' in all_files not found in _find_files_in_timerange. This indicates a drift between docs and implementation." ) - assert 'elif all("P1M" in s for s in all_files):' in base_code, ( + assert 'elif all("P1M" in s for s in all_files):' in utils_code, ( "Expected check for 'P1M' in all_files not found in _find_files_in_timerange. This indicates a drift between docs and implementation." ) From 9cb27a7c7e452535dd1b9357d06779a0fda3d3f1 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 20 Nov 2025 14:38:38 +0100 Subject: [PATCH 90/97] fix xbt bug --- src/virtualship/instruments/xbt.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index 3e11b55dd..0d6f6276f 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -8,7 +8,7 @@ from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime -from virtualship.utils import register_instrument +from virtualship.utils import add_dummy_UV, register_instrument # ===================================================== # SECTION: Dataclass @@ -79,7 +79,7 @@ class XBTInstrument(Instrument): def __init__(self, expedition, from_data): """Initialize XBTInstrument.""" - variables = {"U": "uo", "V": "vo", "S": "so", "T": "thetao"} + variables = {"T": "thetao"} super().__init__( expedition, variables, @@ -94,7 +94,7 @@ def __init__(self, expedition, from_data): def simulate(self, measurements, out_path) -> None: """Simulate XBT measurements.""" DT = 10.0 # dt of XBT simulation integrator - OUTPUT_DT = timedelta(seconds=1) + OUTPUT_DT = timedelta(seconds=10) if len(measurements) == 0: print( @@ -105,8 +105,15 @@ def simulate(self, measurements, out_path) -> None: fieldset = self.load_input_data() - fieldset_starttime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[0]) - fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) + # add dummy U + add_dummy_UV(fieldset) # TODO: parcels v3 bodge; remove when parcels v4 is used + + fieldset_starttime = fieldset.T.grid.time_origin.fulltime( + fieldset.T.grid.time_full[0] + ) + fieldset_endtime = fieldset.T.grid.time_origin.fulltime( + fieldset.T.grid.time_full[-1] + ) # deploy time for all xbts should be later than fieldset start time if not all( From 6b6d39509d1baab777527e997bd09bf861ba703d Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 20 Nov 2025 15:07:59 +0100 Subject: [PATCH 91/97] add warnings to ADCP max depth config if exceeds authentic limits --- src/virtualship/instruments/adcp.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index 0270b3e68..22f9859b6 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -71,7 +71,18 @@ def __init__(self, expedition, from_data): def simulate(self, measurements, out_path) -> None: """Simulate ADCP measurements.""" - MAX_DEPTH = self.expedition.instruments_config.adcp_config.max_depth_meter + config_max_depth = ( + self.expedition.instruments_config.adcp_config.max_depth_meter + ) + + if config_max_depth < -1600.0: + print( + f"\n\n⚠️ Warning: The configured ADCP max depth of {abs(config_max_depth)} m exceeds the 1600 m limit for the technology (e.g. https://www.geomar.de/en/research/fb1/fb1-po/observing-systems/adcp)." + "\n\n This expedition will continue using the prescribed configuration. However, note, the results will not necessarily represent authentic ADCP instrument readings and could also lead to slower simulations ." + "\n\n If this was unintented, consider re-adjusting your ADCP configuration in your expedition.yaml or via `virtualship plan`.\n\n" + ) + + MAX_DEPTH = config_max_depth MIN_DEPTH = -5.0 NUM_BINS = self.expedition.instruments_config.adcp_config.num_bins From b26636bfb62052998ec9a5d08e2e33eacbc58a0f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 20 Nov 2025 14:11:18 +0000 Subject: [PATCH 92/97] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/virtualship/instruments/adcp.py | 2 +- src/virtualship/instruments/argo_float.py | 2 +- src/virtualship/instruments/base.py | 2 +- src/virtualship/instruments/ctd.py | 2 +- src/virtualship/instruments/ctd_bgc.py | 2 +- src/virtualship/instruments/drifter.py | 2 +- src/virtualship/instruments/ship_underwater_st.py | 2 +- src/virtualship/instruments/xbt.py | 2 +- src/virtualship/utils.py | 2 +- tests/instruments/test_adcp.py | 2 +- tests/instruments/test_argo_float.py | 2 +- tests/instruments/test_ctd.py | 2 +- tests/instruments/test_ctd_bgc.py | 2 +- tests/instruments/test_drifter.py | 2 +- tests/instruments/test_ship_underwater_st.py | 2 +- tests/instruments/test_xbt.py | 2 +- tests/test_utils.py | 2 +- 17 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/virtualship/instruments/adcp.py b/src/virtualship/instruments/adcp.py index 22f9859b6..2a761e14e 100644 --- a/src/virtualship/instruments/adcp.py +++ b/src/virtualship/instruments/adcp.py @@ -2,8 +2,8 @@ from typing import ClassVar import numpy as np - from parcels import ParticleSet, ScipyParticle, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.utils import ( diff --git a/src/virtualship/instruments/argo_float.py b/src/virtualship/instruments/argo_float.py index 2253c1a12..204f0b3dd 100644 --- a/src/virtualship/instruments/argo_float.py +++ b/src/virtualship/instruments/argo_float.py @@ -4,7 +4,6 @@ from typing import ClassVar import numpy as np - from parcels import ( AdvectionRK4, JITParticle, @@ -12,6 +11,7 @@ StatusCode, Variable, ) + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/base.py b/src/virtualship/instruments/base.py index 5faf9a760..22b0b54ac 100644 --- a/src/virtualship/instruments/base.py +++ b/src/virtualship/instruments/base.py @@ -8,9 +8,9 @@ import copernicusmarine import xarray as xr +from parcels import FieldSet from yaspin import yaspin -from parcels import FieldSet from virtualship.errors import CopernicusCatalogueError from virtualship.utils import ( COPERNICUSMARINE_PHYS_VARIABLES, diff --git a/src/virtualship/instruments/ctd.py b/src/virtualship/instruments/ctd.py index f90c5fc0c..73248cf93 100644 --- a/src/virtualship/instruments/ctd.py +++ b/src/virtualship/instruments/ctd.py @@ -3,8 +3,8 @@ from typing import TYPE_CHECKING, ClassVar import numpy as np - from parcels import JITParticle, ParticleSet, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py index 039e850c2..fab9e07b5 100644 --- a/src/virtualship/instruments/ctd_bgc.py +++ b/src/virtualship/instruments/ctd_bgc.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np - from parcels import JITParticle, ParticleSet, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/drifter.py b/src/virtualship/instruments/drifter.py index 8428ad7cf..e962278d6 100644 --- a/src/virtualship/instruments/drifter.py +++ b/src/virtualship/instruments/drifter.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np - from parcels import AdvectionRK4, JITParticle, ParticleSet, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/instruments/ship_underwater_st.py b/src/virtualship/instruments/ship_underwater_st.py index e6af25b31..088a439f9 100644 --- a/src/virtualship/instruments/ship_underwater_st.py +++ b/src/virtualship/instruments/ship_underwater_st.py @@ -2,8 +2,8 @@ from typing import ClassVar import numpy as np - from parcels import ParticleSet, ScipyParticle, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.utils import add_dummy_UV, register_instrument diff --git a/src/virtualship/instruments/xbt.py b/src/virtualship/instruments/xbt.py index 0d6f6276f..f0f5d130a 100644 --- a/src/virtualship/instruments/xbt.py +++ b/src/virtualship/instruments/xbt.py @@ -3,8 +3,8 @@ from typing import ClassVar import numpy as np - from parcels import JITParticle, ParticleSet, Variable + from virtualship.instruments.base import Instrument from virtualship.instruments.types import InstrumentType from virtualship.models.spacetime import Spacetime diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index 6dc25cde7..7e37617b2 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -13,8 +13,8 @@ import copernicusmarine import numpy as np import xarray as xr - from parcels import FieldSet + from virtualship.errors import CopernicusCatalogueError if TYPE_CHECKING: diff --git a/tests/instruments/test_adcp.py b/tests/instruments/test_adcp.py index d87dae423..a2a5418a9 100644 --- a/tests/instruments/test_adcp.py +++ b/tests/instruments/test_adcp.py @@ -4,8 +4,8 @@ import numpy as np import xarray as xr - from parcels import FieldSet + from virtualship.instruments.adcp import ADCPInstrument from virtualship.models import Location, Spacetime diff --git a/tests/instruments/test_argo_float.py b/tests/instruments/test_argo_float.py index 1dfa4060b..cbe25d764 100644 --- a/tests/instruments/test_argo_float.py +++ b/tests/instruments/test_argo_float.py @@ -4,8 +4,8 @@ import numpy as np import xarray as xr - from parcels import FieldSet + from virtualship.instruments.argo_float import ArgoFloat, ArgoFloatInstrument from virtualship.models import Location, Spacetime diff --git a/tests/instruments/test_ctd.py b/tests/instruments/test_ctd.py index 0382e1597..fff5fc4fd 100644 --- a/tests/instruments/test_ctd.py +++ b/tests/instruments/test_ctd.py @@ -8,8 +8,8 @@ import numpy as np import xarray as xr - from parcels import Field, FieldSet + from virtualship.instruments.ctd import CTD, CTDInstrument from virtualship.models import Location, Spacetime diff --git a/tests/instruments/test_ctd_bgc.py b/tests/instruments/test_ctd_bgc.py index 7d1676ed8..00f300777 100644 --- a/tests/instruments/test_ctd_bgc.py +++ b/tests/instruments/test_ctd_bgc.py @@ -8,8 +8,8 @@ import numpy as np import xarray as xr - from parcels import Field, FieldSet + from virtualship.instruments.ctd_bgc import CTD_BGC, CTD_BGCInstrument from virtualship.models import Location, Spacetime diff --git a/tests/instruments/test_drifter.py b/tests/instruments/test_drifter.py index 27bc7ffdd..9253c1a81 100644 --- a/tests/instruments/test_drifter.py +++ b/tests/instruments/test_drifter.py @@ -4,8 +4,8 @@ import numpy as np import xarray as xr - from parcels import FieldSet + from virtualship.instruments.drifter import Drifter, DrifterInstrument from virtualship.models import Location, Spacetime diff --git a/tests/instruments/test_ship_underwater_st.py b/tests/instruments/test_ship_underwater_st.py index 96778f87f..e7ca18d10 100644 --- a/tests/instruments/test_ship_underwater_st.py +++ b/tests/instruments/test_ship_underwater_st.py @@ -4,8 +4,8 @@ import numpy as np import xarray as xr - from parcels import FieldSet + from virtualship.instruments.ship_underwater_st import Underwater_STInstrument from virtualship.models import Location, Spacetime diff --git a/tests/instruments/test_xbt.py b/tests/instruments/test_xbt.py index 524125101..d218025a8 100644 --- a/tests/instruments/test_xbt.py +++ b/tests/instruments/test_xbt.py @@ -8,8 +8,8 @@ import numpy as np import xarray as xr - from parcels import Field, FieldSet + from virtualship.instruments.xbt import XBT, XBTInstrument from virtualship.models import Location, Spacetime diff --git a/tests/test_utils.py b/tests/test_utils.py index 6004ea69c..8bd2338eb 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -3,9 +3,9 @@ import numpy as np import pytest import xarray as xr +from parcels import FieldSet import virtualship.utils -from parcels import FieldSet from virtualship.models.expedition import Expedition from virtualship.utils import ( _find_nc_file_with_variable, From ac02bdcf9951b3fb5c557fbee896a10c6f97d542 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 20 Nov 2025 15:42:16 +0100 Subject: [PATCH 93/97] error messaging for case where measurements cause schedule to be missed --- src/virtualship/expedition/simulate_schedule.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/virtualship/expedition/simulate_schedule.py b/src/virtualship/expedition/simulate_schedule.py index 1d43dc13b..06e0c2fc4 100644 --- a/src/virtualship/expedition/simulate_schedule.py +++ b/src/virtualship/expedition/simulate_schedule.py @@ -8,6 +8,7 @@ import pyproj +from virtualship.errors import ScheduleError from virtualship.instruments.argo_float import ArgoFloat from virtualship.instruments.ctd import CTD from virtualship.instruments.ctd_bgc import CTD_BGC @@ -121,8 +122,9 @@ def simulate(self) -> ScheduleOk | ScheduleProblem: # check if waypoint was reached in time if waypoint.time is not None and self._time > waypoint.time: - print( + raise ScheduleError( f"Waypoint {wp_i + 1} could not be reached in time. Current time: {self._time}. Waypoint time: {waypoint.time}." + "\n\nHave you ensured that your schedule includes sufficient time for taking measurements such as CTD casts (in addition to the time it takes to sail between waypoints)?\n\n" ) return ScheduleProblem(self._time, wp_i) else: From 7b25526079b1c22b9bfb3c039034bbf2cc6e57b4 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Fri, 21 Nov 2025 09:02:24 +0100 Subject: [PATCH 94/97] revert to using ScheduleProblem class --- src/virtualship/expedition/simulate_schedule.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/virtualship/expedition/simulate_schedule.py b/src/virtualship/expedition/simulate_schedule.py index 06e0c2fc4..0513de651 100644 --- a/src/virtualship/expedition/simulate_schedule.py +++ b/src/virtualship/expedition/simulate_schedule.py @@ -8,7 +8,6 @@ import pyproj -from virtualship.errors import ScheduleError from virtualship.instruments.argo_float import ArgoFloat from virtualship.instruments.ctd import CTD from virtualship.instruments.ctd_bgc import CTD_BGC @@ -122,7 +121,7 @@ def simulate(self) -> ScheduleOk | ScheduleProblem: # check if waypoint was reached in time if waypoint.time is not None and self._time > waypoint.time: - raise ScheduleError( + print( f"Waypoint {wp_i + 1} could not be reached in time. Current time: {self._time}. Waypoint time: {waypoint.time}." "\n\nHave you ensured that your schedule includes sufficient time for taking measurements such as CTD casts (in addition to the time it takes to sail between waypoints)?\n\n" ) From 28dbe3397f3a3b7b5edf049fbcbd178a5ec82db6 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Fri, 21 Nov 2025 09:11:42 +0100 Subject: [PATCH 95/97] add more informative messaging on ScheduleProblem --- src/virtualship/cli/_run.py | 2 +- src/virtualship/expedition/simulate_schedule.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/virtualship/cli/_run.py b/src/virtualship/cli/_run.py index 6ae7a1cd1..f07fbab27 100644 --- a/src/virtualship/cli/_run.py +++ b/src/virtualship/cli/_run.py @@ -98,7 +98,7 @@ def _run(expedition_dir: str | Path, from_data: Path | None = None) -> None: ) if isinstance(schedule_results, ScheduleProblem): print( - "Update your schedule and continue the expedition by running the tool again." + f"SIMULATION PAUSED: update your schedule (`virtualship plan`) and continue the expedition by executing the `virtualship run` command again.\nCheckpoint has been saved to {expedition_dir.joinpath(CHECKPOINT)}." ) _save_checkpoint( Checkpoint( diff --git a/src/virtualship/expedition/simulate_schedule.py b/src/virtualship/expedition/simulate_schedule.py index 0513de651..0a567b1c6 100644 --- a/src/virtualship/expedition/simulate_schedule.py +++ b/src/virtualship/expedition/simulate_schedule.py @@ -123,7 +123,8 @@ def simulate(self) -> ScheduleOk | ScheduleProblem: if waypoint.time is not None and self._time > waypoint.time: print( f"Waypoint {wp_i + 1} could not be reached in time. Current time: {self._time}. Waypoint time: {waypoint.time}." - "\n\nHave you ensured that your schedule includes sufficient time for taking measurements such as CTD casts (in addition to the time it takes to sail between waypoints)?\n\n" + "\n\nHave you ensured that your schedule includes sufficient time for taking measurements, e.g. CTD casts (in addition to the time it takes to sail between waypoints)?\n" + "**Note**, the `virtualship plan` tool will not account for measurement times when verifying the schedule, only the time it takes to sail between waypoints.\n" ) return ScheduleProblem(self._time, wp_i) else: From 9bf6446c705a87d8cf60519bdc831585339d850b Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Fri, 21 Nov 2025 09:49:50 +0100 Subject: [PATCH 96/97] change test to mock using data from disk to avoid copernicus calls --- tests/cli/test_run.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/cli/test_run.py b/tests/cli/test_run.py index 5ad14c648..e2930dba1 100644 --- a/tests/cli/test_run.py +++ b/tests/cli/test_run.py @@ -50,10 +50,13 @@ def test_run(tmp_path, monkeypatch): "virtualship.cli._run.get_instrument_class", lambda itype: DummyInstrument ) - fake_data_dir = None + fake_data_dir = tmp_path / "fake_data" + fake_data_dir.mkdir() _run(expedition_dir, from_data=fake_data_dir) + breakpoint() + results_dir = expedition_dir / "results" assert results_dir.exists() and results_dir.is_dir() From 0d5c5eb0c49049a22d713283e4fd89a52f95631f Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Fri, 21 Nov 2025 09:54:04 +0100 Subject: [PATCH 97/97] remove accidental breakpoint --- tests/cli/test_run.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/cli/test_run.py b/tests/cli/test_run.py index e2930dba1..190442347 100644 --- a/tests/cli/test_run.py +++ b/tests/cli/test_run.py @@ -55,8 +55,6 @@ def test_run(tmp_path, monkeypatch): _run(expedition_dir, from_data=fake_data_dir) - breakpoint() - results_dir = expedition_dir / "results" assert results_dir.exists() and results_dir.is_dir()