From 221a0edfc446b92869111ac3424458e5d1c9a15c Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 30 Apr 2025 15:13:08 +0200 Subject: [PATCH 01/15] update init command to use CTD_BGC as new instrument when making schedule --- src/virtualship/expedition/instrument_type.py | 1 + src/virtualship/utils.py | 1 + 2 files changed, 2 insertions(+) diff --git a/src/virtualship/expedition/instrument_type.py b/src/virtualship/expedition/instrument_type.py index 82360c7b..43ffe68d 100644 --- a/src/virtualship/expedition/instrument_type.py +++ b/src/virtualship/expedition/instrument_type.py @@ -7,6 +7,7 @@ class InstrumentType(Enum): """Types of instruments.""" CTD = "CTD" + CTD_BGC = "CTD_BGC" DRIFTER = "DRIFTER" ARGO_FLOAT = "ARGO_FLOAT" XBT = "XBT" diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index c798a3dd..848d5d63 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -157,6 +157,7 @@ def mfp_to_yaml(coordinates_file_path: str, yaml_output_path: str): # noqa: D41 instrument_max_depths = { "XBT": 2000, "CTD": 5000, + "CTD_BGC": 5000, "DRIFTER": 1, "ARGO_FLOAT": 2000, } From ff28801f877af66907862aa30ff51aea1f37b961 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Thu, 1 May 2025 14:51:06 +0200 Subject: [PATCH 02/15] update example ship_config file to also include CTD_BGC instrument config --- src/virtualship/static/ship_config.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/virtualship/static/ship_config.yaml b/src/virtualship/static/ship_config.yaml index 5066e38d..34d6c6ea 100644 --- a/src/virtualship/static/ship_config.yaml +++ b/src/virtualship/static/ship_config.yaml @@ -14,6 +14,10 @@ ctd_config: max_depth_meter: -2000.0 min_depth_meter: -11.0 stationkeeping_time_minutes: 20.0 +ctd_bgc_config: + max_depth_meter: -2000.0 + min_depth_meter: -11.0 + stationkeeping_time_minutes: 20.0 drifter_config: depth_meter: 0.0 lifetime_minutes: 60480.0 From 9464a17d6aac6e6a3a4297d3c16c7177d2f1096d Mon Sep 17 00:00:00 2001 From: Jamie Atkins <106238905+j-atkins@users.noreply.github.com> Date: Fri, 2 May 2025 13:28:46 +0200 Subject: [PATCH 03/15] Prevent fetch from downloading unnecessary data and skipping data download for certain instruments (#175) * fixes for over-extensive data downloads and skipping certain instruments * fixes based on PR feedback #175 --- src/virtualship/cli/_fetch.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/virtualship/cli/_fetch.py b/src/virtualship/cli/_fetch.py index 340edfd9..16712d38 100644 --- a/src/virtualship/cli/_fetch.py +++ b/src/virtualship/cli/_fetch.py @@ -38,6 +38,8 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None be provided on prompt, via command line arguments, or via a YAML config file. Run `virtualship fetch` on an expedition for more info. """ + from virtualship.expedition.instrument_type import InstrumentType + if sum([username is None, password is None]) == 1: raise ValueError("Both username and password must be provided when using CLI.") @@ -88,10 +90,10 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None {"XBT", "CTD", "SHIP_UNDERWATER_ST"} & set(instrument.name for instrument in instruments_in_schedule) ) - or hasattr(ship_config, "ship_underwater_st_config") - or hasattr(ship_config, "adcp_config") + or ship_config.ship_underwater_st_config is not None + or ship_config.adcp_config is not None ): - print("Ship data will be downloaded") + print("Ship data will be downloaded. Please wait...") # Define all ship datasets to download, including bathymetry download_dict = { @@ -145,8 +147,8 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None complete_download(download_folder) click.echo("Ship data download based on space-time region completed.") - if "DRIFTER" in instruments_in_schedule: - print("Drifter data will be downloaded") + if InstrumentType.DRIFTER in instruments_in_schedule: + print("Drifter data will be downloaded. Please wait...") drifter_download_dict = { "UVdata": { "dataset_id": "cmems_mod_glo_phy-cur_anfc_0.083deg_PT6H-i", @@ -188,8 +190,8 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None complete_download(download_folder) click.echo("Drifter data download based on space-time region completed.") - if "ARGO_FLOAT" in instruments_in_schedule: - print("Argo float data will be downloaded") + if InstrumentType.ARGO_FLOAT in instruments_in_schedule: + print("Argo float data will be downloaded. Please wait...") argo_download_dict = { "UVdata": { "dataset_id": "cmems_mod_glo_phy-cur_anfc_0.083deg_PT6H-i", From 91a28d633fefa1947fe1b3d8ae3d946cd3d3a763 Mon Sep 17 00:00:00 2001 From: Vecko <36369090+VeckoTheGecko@users.noreply.github.com> Date: Fri, 2 May 2025 15:38:29 +0200 Subject: [PATCH 04/15] Managing conflicts on branch. Move InstrumentType to ship_config.py --- src/virtualship/cli/_fetch.py | 2 +- src/virtualship/expedition/__init__.py | 1 - src/virtualship/expedition/checkpoint.py | 2 +- src/virtualship/expedition/instrument_type.py | 13 ------------- src/virtualship/expedition/schedule.py | 2 +- src/virtualship/expedition/ship_config.py | 15 +++++++++++++-- src/virtualship/expedition/simulate_schedule.py | 3 +-- src/virtualship/expedition/waypoint.py | 10 ++++++---- src/virtualship/utils.py | 2 +- tests/test_mfp_to_yaml.py | 2 +- 10 files changed, 25 insertions(+), 27 deletions(-) delete mode 100644 src/virtualship/expedition/instrument_type.py diff --git a/src/virtualship/cli/_fetch.py b/src/virtualship/cli/_fetch.py index 16712d38..2e09dec2 100644 --- a/src/virtualship/cli/_fetch.py +++ b/src/virtualship/cli/_fetch.py @@ -38,7 +38,7 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None be provided on prompt, via command line arguments, or via a YAML config file. Run `virtualship fetch` on an expedition for more info. """ - from virtualship.expedition.instrument_type import InstrumentType + from virtualship.expedition.ship_config import InstrumentType if sum([username is None, password is None]) == 1: raise ValueError("Both username and password must be provided when using CLI.") diff --git a/src/virtualship/expedition/__init__.py b/src/virtualship/expedition/__init__.py index 9137f7b1..831371bd 100644 --- a/src/virtualship/expedition/__init__.py +++ b/src/virtualship/expedition/__init__.py @@ -2,7 +2,6 @@ from .do_expedition import do_expedition from .input_data import InputData -from .instrument_type import InstrumentType from .schedule import Schedule from .ship_config import ( ADCPConfig, diff --git a/src/virtualship/expedition/checkpoint.py b/src/virtualship/expedition/checkpoint.py index 75dd2356..be6079ef 100644 --- a/src/virtualship/expedition/checkpoint.py +++ b/src/virtualship/expedition/checkpoint.py @@ -7,8 +7,8 @@ import pydantic import yaml -from .instrument_type import InstrumentType from .schedule import Schedule +from .ship_config import InstrumentType class _YamlDumper(yaml.SafeDumper): diff --git a/src/virtualship/expedition/instrument_type.py b/src/virtualship/expedition/instrument_type.py deleted file mode 100644 index 43ffe68d..00000000 --- a/src/virtualship/expedition/instrument_type.py +++ /dev/null @@ -1,13 +0,0 @@ -"""InstrumentType Enum.""" - -from enum import Enum - - -class InstrumentType(Enum): - """Types of instruments.""" - - CTD = "CTD" - CTD_BGC = "CTD_BGC" - DRIFTER = "DRIFTER" - ARGO_FLOAT = "ARGO_FLOAT" - XBT = "XBT" diff --git a/src/virtualship/expedition/schedule.py b/src/virtualship/expedition/schedule.py index 6c3bbfd1..5658f8d2 100644 --- a/src/virtualship/expedition/schedule.py +++ b/src/virtualship/expedition/schedule.py @@ -12,7 +12,7 @@ from parcels import FieldSet from .input_data import InputData -from .instrument_type import InstrumentType +from .ship_config import InstrumentType from .space_time_region import SpaceTimeRegion from .waypoint import Waypoint diff --git a/src/virtualship/expedition/ship_config.py b/src/virtualship/expedition/ship_config.py index b411443f..4f238b36 100644 --- a/src/virtualship/expedition/ship_config.py +++ b/src/virtualship/expedition/ship_config.py @@ -3,15 +3,26 @@ from __future__ import annotations from datetime import timedelta +from enum import Enum from pathlib import Path +from typing import TYPE_CHECKING import pydantic import yaml from virtualship.utils import _validate_numeric_mins_to_timedelta -from .instrument_type import InstrumentType -from .schedule import Schedule +if TYPE_CHECKING: + from .schedule import Schedule + + +class InstrumentType(Enum): + """Types of instruments.""" + + CTD = "CTD" + DRIFTER = "DRIFTER" + ARGO_FLOAT = "ARGO_FLOAT" + XBT = "XBT" class ArgoFloatConfig(pydantic.BaseModel): diff --git a/src/virtualship/expedition/simulate_schedule.py b/src/virtualship/expedition/simulate_schedule.py index fadfd2aa..db22bd71 100644 --- a/src/virtualship/expedition/simulate_schedule.py +++ b/src/virtualship/expedition/simulate_schedule.py @@ -13,9 +13,8 @@ from ..instruments.xbt import XBT from ..location import Location from ..spacetime import Spacetime -from .instrument_type import InstrumentType from .schedule import Schedule -from .ship_config import ShipConfig +from .ship_config import InstrumentType, ShipConfig from .waypoint import Waypoint diff --git a/src/virtualship/expedition/waypoint.py b/src/virtualship/expedition/waypoint.py index 018ccecb..9fb08a52 100644 --- a/src/virtualship/expedition/waypoint.py +++ b/src/virtualship/expedition/waypoint.py @@ -1,21 +1,23 @@ """Waypoint class.""" +from __future__ import annotations + from datetime import datetime -from pydantic import BaseModel, field_serializer +import pydantic from ..location import Location -from .instrument_type import InstrumentType +from .ship_config import InstrumentType -class Waypoint(BaseModel): +class Waypoint(pydantic.BaseModel): """A Waypoint to sail to with an optional time and an optional instrument.""" location: Location time: datetime | None = None instrument: InstrumentType | list[InstrumentType] | None = None - @field_serializer("instrument") + @pydantic.field_serializer("instrument") def serialize_instrument(self, instrument): """Ensure InstrumentType is serialized as a string (or list of strings).""" if isinstance(instrument, list): diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index 848d5d63..5aa85f26 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -139,8 +139,8 @@ def mfp_to_yaml(coordinates_file_path: str, yaml_output_path: str): # noqa: D41 """ # Importing Schedule and related models from expedition module - from virtualship.expedition.instrument_type import InstrumentType from virtualship.expedition.schedule import Schedule + from virtualship.expedition.ship_config import InstrumentType from virtualship.expedition.space_time_region import ( SpaceTimeRegion, SpatialRange, diff --git a/tests/test_mfp_to_yaml.py b/tests/test_mfp_to_yaml.py index a3175185..3ef511f5 100644 --- a/tests/test_mfp_to_yaml.py +++ b/tests/test_mfp_to_yaml.py @@ -3,8 +3,8 @@ import pandas as pd import pytest -from virtualship.expedition.instrument_type import InstrumentType from virtualship.expedition.schedule import Schedule +from virtualship.expedition.ship_config import InstrumentType from virtualship.utils import mfp_to_yaml From 1ae51d3f41bded49cff2eb4bc07f0cace0fbc21e Mon Sep 17 00:00:00 2001 From: Vecko <36369090+VeckoTheGecko@users.noreply.github.com> Date: Fri, 2 May 2025 15:48:37 +0200 Subject: [PATCH 05/15] Move Waypoint to schedule.py --- src/virtualship/expedition/__init__.py | 3 +-- src/virtualship/expedition/schedule.py | 19 ++++++++++++-- .../expedition/simulate_schedule.py | 3 +-- src/virtualship/expedition/waypoint.py | 25 ------------------- src/virtualship/utils.py | 3 +-- tests/expedition/test_schedule.py | 3 +-- 6 files changed, 21 insertions(+), 35 deletions(-) delete mode 100644 src/virtualship/expedition/waypoint.py diff --git a/src/virtualship/expedition/__init__.py b/src/virtualship/expedition/__init__.py index 831371bd..74403732 100644 --- a/src/virtualship/expedition/__init__.py +++ b/src/virtualship/expedition/__init__.py @@ -2,7 +2,7 @@ from .do_expedition import do_expedition from .input_data import InputData -from .schedule import Schedule +from .schedule import Schedule, Waypoint from .ship_config import ( ADCPConfig, ArgoFloatConfig, @@ -12,7 +12,6 @@ ShipUnderwaterSTConfig, ) from .space_time_region import SpaceTimeRegion -from .waypoint import Waypoint __all__ = [ "ADCPConfig", diff --git a/src/virtualship/expedition/schedule.py b/src/virtualship/expedition/schedule.py index 5658f8d2..79f14758 100644 --- a/src/virtualship/expedition/schedule.py +++ b/src/virtualship/expedition/schedule.py @@ -3,7 +3,7 @@ from __future__ import annotations import itertools -from datetime import timedelta +from datetime import datetime, timedelta from pathlib import Path import pydantic @@ -11,14 +11,29 @@ import yaml from parcels import FieldSet +from ..location import Location from .input_data import InputData from .ship_config import InstrumentType from .space_time_region import SpaceTimeRegion -from .waypoint import Waypoint projection: pyproj.Geod = pyproj.Geod(ellps="WGS84") +class Waypoint(pydantic.BaseModel): + """A Waypoint to sail to with an optional time and an optional instrument.""" + + location: Location + time: datetime | None = None + instrument: InstrumentType | list[InstrumentType] | None = None + + @pydantic.field_serializer("instrument") + def serialize_instrument(self, instrument): + """Ensure InstrumentType is serialized as a string (or list of strings).""" + if isinstance(instrument, list): + return [inst.value for inst in instrument] + return instrument.value if instrument else None + + class Schedule(pydantic.BaseModel): """Schedule of the virtual ship.""" diff --git a/src/virtualship/expedition/simulate_schedule.py b/src/virtualship/expedition/simulate_schedule.py index db22bd71..df36b8b2 100644 --- a/src/virtualship/expedition/simulate_schedule.py +++ b/src/virtualship/expedition/simulate_schedule.py @@ -13,9 +13,8 @@ from ..instruments.xbt import XBT from ..location import Location from ..spacetime import Spacetime -from .schedule import Schedule +from .schedule import Schedule, Waypoint from .ship_config import InstrumentType, ShipConfig -from .waypoint import Waypoint @dataclass diff --git a/src/virtualship/expedition/waypoint.py b/src/virtualship/expedition/waypoint.py deleted file mode 100644 index 9fb08a52..00000000 --- a/src/virtualship/expedition/waypoint.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Waypoint class.""" - -from __future__ import annotations - -from datetime import datetime - -import pydantic - -from ..location import Location -from .ship_config import InstrumentType - - -class Waypoint(pydantic.BaseModel): - """A Waypoint to sail to with an optional time and an optional instrument.""" - - location: Location - time: datetime | None = None - instrument: InstrumentType | list[InstrumentType] | None = None - - @pydantic.field_serializer("instrument") - def serialize_instrument(self, instrument): - """Ensure InstrumentType is serialized as a string (or list of strings).""" - if isinstance(instrument, list): - return [inst.value for inst in instrument] - return instrument.value if instrument else None diff --git a/src/virtualship/utils.py b/src/virtualship/utils.py index 5aa85f26..6dbbb49c 100644 --- a/src/virtualship/utils.py +++ b/src/virtualship/utils.py @@ -139,14 +139,13 @@ def mfp_to_yaml(coordinates_file_path: str, yaml_output_path: str): # noqa: D41 """ # Importing Schedule and related models from expedition module - from virtualship.expedition.schedule import Schedule + from virtualship.expedition.schedule import Location, Schedule, Waypoint from virtualship.expedition.ship_config import InstrumentType from virtualship.expedition.space_time_region import ( SpaceTimeRegion, SpatialRange, TimeRange, ) - from virtualship.expedition.waypoint import Location, Waypoint # Read data from file coordinates_data = load_coordinates(coordinates_file_path) diff --git a/tests/expedition/test_schedule.py b/tests/expedition/test_schedule.py index 50c35d2f..eabcbd9a 100644 --- a/tests/expedition/test_schedule.py +++ b/tests/expedition/test_schedule.py @@ -5,9 +5,8 @@ import pytest from virtualship import Location -from virtualship.expedition import Waypoint from virtualship.expedition.do_expedition import _load_input_data -from virtualship.expedition.schedule import Schedule, ScheduleError +from virtualship.expedition.schedule import Schedule, ScheduleError, Waypoint from virtualship.utils import _get_ship_config projection = pyproj.Geod(ellps="WGS84") From 3aed6850e46492e9e9293f8da827753fb332b8c0 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Wed, 30 Apr 2025 15:13:08 +0200 Subject: [PATCH 06/15] update init command to use CTD_BGC as new instrument when making schedule --- src/virtualship/expedition/ship_config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/virtualship/expedition/ship_config.py b/src/virtualship/expedition/ship_config.py index 4f238b36..062b3784 100644 --- a/src/virtualship/expedition/ship_config.py +++ b/src/virtualship/expedition/ship_config.py @@ -20,6 +20,7 @@ class InstrumentType(Enum): """Types of instruments.""" CTD = "CTD" + CTD_BGC = "CTD_BGC" DRIFTER = "DRIFTER" ARGO_FLOAT = "ARGO_FLOAT" XBT = "XBT" From 938052c9aaeae148d2b0eb3d43d5193cf3cb9087 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Fri, 9 May 2025 10:54:57 +0200 Subject: [PATCH 07/15] add configuration for CTD_BGC --- src/virtualship/expedition/ship_config.py | 37 +++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/virtualship/expedition/ship_config.py b/src/virtualship/expedition/ship_config.py index 062b3784..c22b673e 100644 --- a/src/virtualship/expedition/ship_config.py +++ b/src/virtualship/expedition/ship_config.py @@ -81,6 +81,28 @@ def _validate_stationkeeping_time(cls, value: int | float | timedelta) -> timede return _validate_numeric_mins_to_timedelta(value) +class CTD_BGCConfig(pydantic.BaseModel): + """Configuration for CTD_BGC instrument.""" + + stationkeeping_time: timedelta = pydantic.Field( + serialization_alias="stationkeeping_time_minutes", + validation_alias="stationkeeping_time_minutes", + gt=timedelta(), + ) + min_depth_meter: float = pydantic.Field(le=0.0) + max_depth_meter: float = pydantic.Field(le=0.0) + + model_config = pydantic.ConfigDict(populate_by_name=True) + + @pydantic.field_serializer("stationkeeping_time") + def _serialize_stationkeeping_time(self, value: timedelta, _info): + return value.total_seconds() / 60.0 + + @pydantic.field_validator("stationkeeping_time", mode="before") + def _validate_stationkeeping_time(cls, value: int | float | timedelta) -> timedelta: + return _validate_numeric_mins_to_timedelta(value) + + class ShipUnderwaterSTConfig(pydantic.BaseModel): """Configuration for underwater ST.""" @@ -160,6 +182,13 @@ class ShipConfig(pydantic.BaseModel): If None, no CTDs can be cast. """ + ctd_bgc_config: CTD_BGCConfig | None = None + """ + CTD_BGC configuration. + + If None, no BGC CTDs can be cast. + """ + ship_underwater_st_config: ShipUnderwaterSTConfig | None = None """ Ship underwater salinity temperature measurementconfiguration. @@ -240,6 +269,7 @@ def verify(self, schedule: Schedule) -> None: "DRIFTER", "XBT", "CTD", + "CTD_BGC", ]: # TODO make instrument names consistent capitals or lowercase throughout codebase if hasattr(self, instrument.lower() + "_config") and not any( instrument == schedule_instrument.name @@ -249,6 +279,7 @@ def verify(self, schedule: Schedule) -> None: setattr(self, instrument.lower() + "_config", None) # verify instruments in schedule have configuration + # TODO: the ConfigError message could be improved to explain that the **schedule** file has X instrument but the **ship_config** file does not for instrument in instruments_in_schedule: try: InstrumentType(instrument) @@ -267,6 +298,12 @@ def verify(self, schedule: Schedule) -> None: raise ConfigError( "Planning has a waypoint with CTD instrument, but configuration does not configure CTDs." ) + if instrument == InstrumentType.CTD_BGC and ( + not hasattr(self, "ctd_bgc_config") or self.ctd_bgc_config is None + ): + raise ConfigError( + "Planning has a waypoint with CTD_BGC instrument, but configuration does not configure CTD_BGCs." + ) if instrument == InstrumentType.DRIFTER and ( not hasattr(self, "drifter_config") or self.drifter_config is None ): From 21ad1c68b1426343bfcb7ae824d94b887bdba9b2 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Fri, 9 May 2025 11:09:45 +0200 Subject: [PATCH 08/15] add bgc data download option for CTD_BGC instrument --- src/virtualship/cli/_fetch.py | 44 +++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/virtualship/cli/_fetch.py b/src/virtualship/cli/_fetch.py index 2e09dec2..386e6623 100644 --- a/src/virtualship/cli/_fetch.py +++ b/src/virtualship/cli/_fetch.py @@ -238,6 +238,50 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None complete_download(download_folder) click.echo("Argo_float data download based on space-time region completed.") + if InstrumentType.CTD_BGC in instruments_in_schedule: + print("CTD_BGC data will be downloaded. Please wait...") + + ctd_bgc_download_dict = { + "o2data": { + "dataset_id": "cmems_mod_glo_bgc-bio_anfc_0.25deg_P1D-m", + "variables": ["o2"], + "output_filename": "ctd_bgc_o2.nc", + }, + "chlorodata": { + "dataset_id": "cmems_mod_glo_bgc-pft_anfc_0.25deg_P1D-m", + "variables": ["chl"], + "output_filename": "ctd_bgc_chloro.nc", + }, + } + + # Iterate over all datasets and download each based on space_time_region + try: + for dataset in ctd_bgc_download_dict.values(): + copernicusmarine.subset( + dataset_id=dataset["dataset_id"], + variables=dataset["variables"], + minimum_longitude=spatial_range.minimum_longitude - 3.0, + maximum_longitude=spatial_range.maximum_longitude + 3.0, + minimum_latitude=spatial_range.minimum_latitude - 3.0, + maximum_latitude=spatial_range.maximum_latitude + 3.0, + start_datetime=start_datetime, + end_datetime=end_datetime + timedelta(days=21), + minimum_depth=abs(1), + maximum_depth=abs(spatial_range.maximum_depth), + output_filename=dataset["output_filename"], + output_directory=download_folder, + username=username, + password=password, + overwrite=True, + coordinates_selection_method="outside", + ) + except InvalidUsernameOrPassword as e: + shutil.rmtree(download_folder) + raise e + + complete_download(download_folder) + click.echo("CTD_BGC data download based on space-time region completed.") + def _hash(s: str, *, length: int) -> str: """Create a hash of a string.""" From 956e395412541abd0796fb31220279d64aae7a9a Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Fri, 9 May 2025 16:47:44 +0200 Subject: [PATCH 09/15] add CTD_BGC to instruments which prompts ship data download --- src/virtualship/cli/_fetch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/virtualship/cli/_fetch.py b/src/virtualship/cli/_fetch.py index 386e6623..b1cfde2a 100644 --- a/src/virtualship/cli/_fetch.py +++ b/src/virtualship/cli/_fetch.py @@ -87,7 +87,7 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None if ( ( - {"XBT", "CTD", "SHIP_UNDERWATER_ST"} + {"XBT", "CTD", "CDT_BGC", "SHIP_UNDERWATER_ST"} & set(instrument.name for instrument in instruments_in_schedule) ) or ship_config.ship_underwater_st_config is not None From b2b7858a3ddd738baf3fe98d52f963f114160efe Mon Sep 17 00:00:00 2001 From: Vecko <36369090+VeckoTheGecko@users.noreply.github.com> Date: Thu, 15 May 2025 10:33:04 +0200 Subject: [PATCH 10/15] Review feedback on complete_download --- src/virtualship/cli/_fetch.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/virtualship/cli/_fetch.py b/src/virtualship/cli/_fetch.py index b1cfde2a..f07c5b92 100644 --- a/src/virtualship/cli/_fetch.py +++ b/src/virtualship/cli/_fetch.py @@ -144,7 +144,6 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None shutil.rmtree(download_folder) raise e - complete_download(download_folder) click.echo("Ship data download based on space-time region completed.") if InstrumentType.DRIFTER in instruments_in_schedule: @@ -187,7 +186,6 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None shutil.rmtree(download_folder) raise e - complete_download(download_folder) click.echo("Drifter data download based on space-time region completed.") if InstrumentType.ARGO_FLOAT in instruments_in_schedule: @@ -235,7 +233,6 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None shutil.rmtree(download_folder) raise e - complete_download(download_folder) click.echo("Argo_float data download based on space-time region completed.") if InstrumentType.CTD_BGC in instruments_in_schedule: @@ -279,9 +276,10 @@ def _fetch(path: str | Path, username: str | None, password: str | None) -> None shutil.rmtree(download_folder) raise e - complete_download(download_folder) click.echo("CTD_BGC data download based on space-time region completed.") + complete_download(download_folder) + def _hash(s: str, *, length: int) -> str: """Create a hash of a string.""" From fda781979c84539936645a87b7bf2e8174562005 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Mon, 12 May 2025 15:47:14 +0200 Subject: [PATCH 11/15] bgc input data .nc files for testing --- .../expedition_dir/input_data/ctd_bgc_chloro.nc | Bin 0 -> 23313 bytes .../expedition_dir/input_data/ctd_bgc_o2.nc | Bin 0 -> 23323 bytes 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/expedition/expedition_dir/input_data/ctd_bgc_chloro.nc create mode 100644 tests/expedition/expedition_dir/input_data/ctd_bgc_o2.nc diff --git a/tests/expedition/expedition_dir/input_data/ctd_bgc_chloro.nc b/tests/expedition/expedition_dir/input_data/ctd_bgc_chloro.nc new file mode 100644 index 0000000000000000000000000000000000000000..efdc51d381ee74b0fefbdac6c097852fd71e55c3 GIT binary patch literal 23313 zcmeHP33wDm7Ou%~Bq3me1myUnRg^mrP7x_GS0I`MBvD{x9fp|>8J*0)%p`#FfdUGM z2Z)M`2VPtrtl+W-9;*Ug7!(PkkPOMPBHkaXczo#UzE?-5JA@b_Ai7;2p{A=|y{dZe zRn_0Gs(SHgXLjr4PRR+0iA08R6XwIEEY?qXrYX)?a&C*l&;JM2~kLgMABg{P&H4+9BVIb zRxiz!W|~4q2}9Z|;-5sE*cO>vEt{A@60%LyMP|z%zgB$h`!C+bWkGeakftl+6PHCR zlG%;->C4tU?kon4ltmdEdqhYVXtw-XGB`Q)F1zFGMcD-fE|NEtGql`mA=k2*=!TRF ze2`}8nLyrSBJIBEE6{xQxY1(?&V>miwYCj%GJHV@|3Yo*5j5QFv;sn`+Vk;#KryW( zXWawLcoUo7u*}*`&Y-<;v=dqcjZT9UUym80?i3$^-S}k zW!VM5i^yI`OOi3xh@lx7OJZyl2F41Jnk;mXsP5mLdgsbBFQ06_kZ#KD|f%X^oGH&_GVg{EZ(dv4_nA& zrj<^AO?T7<2I;W=VyIQ?hQi0TGX`75II?WDkP(c*O`E4Ktji$H!5Jj+`;Tzuxf?Z8ph2VR(<{`<269=a%EhZp*TX*)Ns)6EsZF79$=&@1!}m= z?^TtYU{NSs0?xTmQ0c+y?P-Ozg>eh3MQT_LdesQyh_f2Dt~eTmti+i3-&m2oG$nC@&Z=UQ8JLS<`Qz3so{u}10pKy*no`w z8JSknh8JLcpn^9cy;BKM=Ue6DU>d_0!hh>X+tfP4Ue1^O25y!bbnW3JxGP0C6M0 z9rTp&fi4G(6L%MCXU$N=AN7}U5)>7(@P%f;j3T&{!j$PoR)!AesZkh=n(UCs*o+cw zOh%$G*LuRfW1GUHW+i9AbPWm|=gM}CbLBf+?%bS$LNWwef{O5YqMnEvB?P^E2>N`m zdl6*wKqBThq+hv_q1QJQF~e_B0tXNOK;OK4`^ZPf^~_osj%K}m-=<s;~tm(G!Ctz4cf&vntx16@j9 zj%)rqlU*+^oaNfFezB_{d4(%G_Bgz@TxKUEZXaGldtF>ZpBh+0?HM(6Pxl&1l^S|k2S`h)q2K*f zO_v<5rVHZLbj$W?dfVD+df>)t8p^MxJCdvEx$F1S9Y^AHZm^1;`}BLKwGS=I;&gv$ zoQ{RRDbYA>7m3qJ)8jNF6sKi@IPK(*)BpJ5bfY^?-<}Ys*N%zPKK3|`XT+(eZJb&@ ztfCW^SJC~gs%XOaO1ksTeKhvj>FvX$HdXY))>X6x{B47FbWN_Jsfkr|N%JatLqZjO z?Z--*bF`A~I#Nl~zo?`e4^~p8vXZVXucX<}SJHYBkRY8}Gcb}e~8mTK5t3yxwIq~}<`QVCBb6Kw}UOE}ICK(!p$LOy4W?%MR8l00eM z`+3~ODr=S)SH_aE?(XzC!b>S#t1|9>)q&51i*+3G=)>KkYyo$tTXp#i@f1pg(57cc+EUh+bbrZxKXq4McA|L~mNepkKXx5WS*r zTQT>aP3nzV=tL#JTrBd?6D!rIh#K2GNjaysGdJl3-! ztCzw$9nKZ(hZ%ks5@;CX0OVppVlWM%Xh?eZT$_j)Y|UL^BOuUMy7?32hPwDI@}t@z$JMdJ7~}h`a!_w1kKup+1-ZkVHd5 zyWp8*>5xzwQv5s;0(fOaLfv#oXbm!jYON5lb9!@;DF84H3B8L64I!cD6IX8p5;HVS z0*ymLLT5y;FQT^yqPGU3Hy)xlEn?6wLqftnh+feb-IVxPL=&N5`~`P0Sdf=9j)%W+q%^h*BpnngGN zTrbT0rb>X}FL~mfF&X+wj}^ZNf8pY+;fkidyp0r_6B1N`POXsYtLmv55)IEEWM42rRlR z;ChQe#y^+e*ef@OXUhvUcZmx%6#~}FJJLyHuASFCQ0%2m6fse7teC-+KtmFEwPwzS zhLOH}?~tzx$3yx!aW*E@kMz^n)*hC>tj(=s1>$at<7pI!&kS)NKmzc(;vnERlaXR1 zN6YC~tZ7fYNRh=yBHIkU9w`bqNN0F@A|{;2AvoK}eGHzF2M!lH`{{C$g1_LH8ag<= z%a?u3@7v_T6DGjDLkE++@xchRij*# z*6W0J+;Be$gN)H-hymRS=->Ngm);N0#yu)_X}-)8@cY~){@@ARnaAquy%?V+T2~l~ z!Y4u;#eoofL85q8AfVA3x5Rn~`VzRf@1Gj>lqjJheK7?-KcRR_r}(|_ISsfaU(Y5P z`c_aNB~z7>z5^TBp~KUzjL;K_xZ(R4Ui?Z4`^JbnROE(F^~$!oJ#K%{9Z@~*8Sv!; zsM9DZ&&(5$)a!QKe@n$X`L$m_V0ZF6LUU5#PhQ2xuM?^jiG%~)4ahtzJVNy*7XhMz>px{$vaC^Gga_RizRy2_I}%u7n@$zIxJDOn~l zPE8^w+yyve|6;qKPuLcR?fUr1W4mNQgpCP9`VM2CBBHm}BqD z&FY2O(oECH7-2{|Mf{V9lTSkCc3V9&NJ2Jd^GoHMk+6mThIh#C;6wnNu z_FtYcpq$J>3I%mTh-dy)*OF!U0*c_$p{9*mL>6HYFya)<2;B83*KLrH6Cj=W#m8$% zCgwQs4UU#AxT!s}EayZ=yY*xhQUB1VMvvpD~Y&KXUC{D9v%f>x-b0uG9P_n>KZ1P5` zmV=Qra`P}O2{rWMagJD{rIKd}B{MXLB>Oqt|Ha>RH@$r06+>U_$Fwp@d|BE4W+RiB zR=PcrK@)*N2CTmr8q~VF@}7;1!Fn-{Z0l{LfHAo0@#=Yr46-O^kiwTwNOZgIvEoGQ z$#N%Bty?ys@6cUOHw~Sj=`6^;=)k89e=q;Cm0Kp7h@qHXlCx&esk1B+z4n7|5<6Hg z&9J z-e?TdR1L)$kNGRZ|jVh&HV5BM)YGhi#rz*vv zs&J$RoO9uj(wo)W#|~=?;}%h?)QB4LsZqueXEkiyXt7^6I&&N@%6wZqpc zD;fwv`(m}omX*ah2FsSJ5uX?OqKxyY-jGgC!=Y)=6QJf*;rApHL{V;j-Y`3SUox)1 zGN>|s#e%@^4Fv0y8sMc=1=OHFs#Hb7HBxY_62%z^SF2&)WVI#`jYaAdN5~tjiw2@5 zVcMYvF2Zmq=JjzIW{X;|$I&Bnd4P`>u#>Kmfs{3ukY}12iAp&jqQZ_1${U!MZzmmj z5ljrsgl1w-;iLnPCkt)MRb1*Wt0*2<=Bc2L5;u^xo`-F|Yzzz-|HUM!Wz!UP;@PRG| zj0<-c8fMLKG!P3+<0L35WZ@4_hZ#k1sf8)igRD#)E>&YN7 z{D(J%NzG19gy|X-Sne)zm%GcT+f!0pQAvhDOHdJhZ_FE2V}ziW4?&+Fb}vF~9!SRg zrt~X!GK~6$B3Af4N?_liA4a|U=Z&M6mG>!J5{VUVyK-6K+^$;-zkKe$g+KM}>Ui(+ zVUDW0iH;M$n&~+3)gs5FS08kI{m3hhtv4KWELxl595?B7=Y6vZoXa|ToE3M@b@qS! zcIW3So^twYcROd@pG05TnMKb~M$+YV7t@@!aXLPIDRpH%N0YzVOCNQ&ca3>ZaZUK= zDA$uWdtE1O)m#%+-R?@h`f*pcYqq-
    w5y3+}6@42VBM`aCmE2YKmxvx!fKR2(= zy=m1o?uyjK?xMHuh0m6AoP?zPgN*22eC5dQ#pG;=3m_idy?EzZjg~*7(P2+%^!z6@ zy6o>7{i$A~>Z2On^sq+zt=8yO4{CJt{TfYKrP1LlH9B#*Myu}un!7bxwp631->K0( zOEfz6R*k;92=cDg;3M@AIO>nj|ID2lz3nZHUjL>>*KOBmkJmMN%QlUkwpF8#Y}RPe zCXK%Hibkit4DEVJqw_XuwCx6s)~(a%-WN1F`gx5md{(1t)@bz9r$HVdgQd{dZJ_KyJ>zt) z5~uH;7N>nrjnk5>I8{yo`jbEo$#HrC^fUdFpR-1HZAKqSo^s#YrQF3TYnB*S#*#8|cls3JrIfB!8Rwk0 z_aotAEk_>xxqFoDPWQtX;0@Jxr(Z+vg}jYycRC#@fE9jaG`>5%0x4kc86DOQyVJe+ z7YEy&UJf~?-RW6a1U9r0^|RgS*YO4HPU8-Dqr1~TtnBzA5?i5F5-_@09fE`Lm;v9N z7CIwFeG#KY5Ti8^qwx@>)DCBR%P^3W42oINyh>(6qr z;E2`T1g&wg;Cz7-y*&5WTrUf+_*i*a&e#1TW-kj)RoL1RaY|DsGrLi6n+wTYYIQ_a zqe@vgf=(IXoo(=@NIE~K^Mp8^@OB>VS&`LCVI2?W3iiVae+UUQjd1{Su^=&+j!<-@ z!t?R$SaU+-7^esBx3IA0vCGCJZZ`KA$nzR7+Jf&_eaMlJY@cYR$;*qS5|L0nEMyoY z0+R9i;=lG2NT?Ed0cdFt5ko@#@dZE<9SOY-X;NrFLYV zA!6tBW+PJoU^)`ojxU-*LTi(kKMEvPXq^O_hlGUAh*4j}Xc5F{4a8_X#AsT?q+f=F zgnbaBqW^Sd@#@xf$12rA2{0rikJYiVxg$C}^0SZ-u9Nx_-x?%@Up>e~zu?haD8y?% zR#50KI>s?WAqm_{I_iE~f@4B*`QcQW@D1k&vui7uNBBLO5Sr z;g28zjwskMedrD@wism0Q->ppsp5#HL=rMC8o2lZ=DzA72yDQ4Ar4D&Ug<+?CA*{C zrjXaMOT!|- z`^c`TbC3dfAj)fSP4zGUvb!P21jyJLoo~1}@gwpGHgR!c@|~;LT?*u6g`-KJdH74H zh#1vlG5}&!J~0{(F)E!H?TnaoYqT$7RPw=#uZ|zB7gm02CBX2PJn_z$3}4rK@z26v zxH#*$qBZynClH*o@%n6Yp%<_8SV6B!9a5R07tHLerIpaqCeGnVGVGAf$qdeL%ECU6 z4cA7ZN<{$9Fe>>Yh7Qafm=FJSaW=p-Szt0N5IwAbIIZdP230&2#10intOmZYYB-&V z#~F|PnIPzt6@If4;8$q&B-vcp#H2Phn*a+07F`x_y~QBoqvbdE%FUsf@_=E1tLEjj)eJf3T;kk9P?+f zO+G9?!Z~x3z~0PtMLCdW*n2M$h>JGtjxxhWqd9KQn6M&~_>iKIk4d&IJRL{pBo5aZ zvSwbdUL1_GE!}&7fB+91IHP7hw1s5i6C7AWSEp~$f)Du3n^H*O3%G&&2?oP^WhI=@ru8o$!?;8ulF)4iK>Zf)elRjBYn=gSebX*Br zRq=Q=1pL;_tUmIN#bSQ4-#U`fD|fF%J-0>@4QZ0}9FxLGxhf=}kPXzvYomN3ZJ&Xdep zkN&-%cm4g)Ox&$v*XO5sg8{#%CJ<`D?Rl)e(JS)k@4(i%0?}wVI8F5{<7U)Vt4F#E zpDoC1;0Ax7CLDxw+~F!wM1kL+P<*wQ1bpzD8gNy Date: Mon, 12 May 2025 15:54:03 +0200 Subject: [PATCH 12/15] add CTD_BGC to static/example schedule (also necessary for ship_config test) --- src/virtualship/static/schedule.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/virtualship/static/schedule.yaml b/src/virtualship/static/schedule.yaml index d15736d0..7cb39423 100644 --- a/src/virtualship/static/schedule.yaml +++ b/src/virtualship/static/schedule.yaml @@ -12,6 +12,7 @@ space_time_region: waypoints: - instrument: - CTD + - CTD_BGC location: latitude: 0 longitude: 0 From 2449163b438f446107fef33957c3e18721613c27 Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 13 May 2025 09:48:42 +0200 Subject: [PATCH 13/15] update test suite for new bgc sampling using ctd_bgc --- .../expedition_dir/ship_config.yaml | 4 + tests/expedition/test_ship_config.py | 12 ++ tests/expedition/test_simulate_schedule.py | 1 + tests/instruments/test_ctd_bgc.py | 137 ++++++++++++++++++ 4 files changed, 154 insertions(+) create mode 100644 tests/instruments/test_ctd_bgc.py diff --git a/tests/expedition/expedition_dir/ship_config.yaml b/tests/expedition/expedition_dir/ship_config.yaml index 09f40c0a..1bae9d1d 100644 --- a/tests/expedition/expedition_dir/ship_config.yaml +++ b/tests/expedition/expedition_dir/ship_config.yaml @@ -14,6 +14,10 @@ ctd_config: max_depth_meter: -2000.0 min_depth_meter: -11.0 stationkeeping_time_minutes: 20.0 +ctd_bgc_config: + max_depth_meter: -2000.0 + min_depth_meter: -11.0 + stationkeeping_time_minutes: 20.0 drifter_config: depth_meter: 0.0 lifetime_minutes: 40320.0 diff --git a/tests/expedition/test_ship_config.py b/tests/expedition/test_ship_config.py index 44bfd524..920058d2 100644 --- a/tests/expedition/test_ship_config.py +++ b/tests/expedition/test_ship_config.py @@ -50,6 +50,12 @@ def ship_config_no_ctd(ship_config): return ship_config +@pytest.fixture +def ship_config_no_ctd_bgc(ship_config): + delattr(ship_config, "ctd_bgc_config") + return ship_config + + @pytest.fixture def ship_config_no_argo_float(ship_config): delattr(ship_config, "argo_float_config") @@ -91,6 +97,12 @@ def test_verify_ship_config_no_instrument(ship_config, schedule_no_xbt) -> None: "Planning has a waypoint with CTD instrument, but configuration does not configure CTD.", id="ShipConfigNoCTD", ), + pytest.param( + "ship_config_no_ctd_bgc", + ConfigError, + "Planning has a waypoint with CTD_BGC instrument, but configuration does not configure CTD_BGCs.", + id="ShipConfigNoCTD_BGC", + ), pytest.param( "ship_config_no_argo_float", ConfigError, diff --git a/tests/expedition/test_simulate_schedule.py b/tests/expedition/test_simulate_schedule.py index 6381b805..8c42097b 100644 --- a/tests/expedition/test_simulate_schedule.py +++ b/tests/expedition/test_simulate_schedule.py @@ -53,4 +53,5 @@ def test_time_in_minutes_in_ship_schedule() -> None: ship_config = ShipConfig.from_yaml("expedition_dir/ship_config.yaml") assert ship_config.adcp_config.period == timedelta(minutes=5) assert ship_config.ctd_config.stationkeeping_time == timedelta(minutes=20) + assert ship_config.ctd_bgc_config.stationkeeping_time == timedelta(minutes=20) assert ship_config.ship_underwater_st_config.period == timedelta(minutes=5) diff --git a/tests/instruments/test_ctd_bgc.py b/tests/instruments/test_ctd_bgc.py new file mode 100644 index 00000000..8083c580 --- /dev/null +++ b/tests/instruments/test_ctd_bgc.py @@ -0,0 +1,137 @@ +""" +Test the simulation of CTD_BGC instruments. + +Fields are kept static over time and time component of CTD_BGC measurements is not tested because it's tricky to provide expected measurements. +""" + +import datetime +from datetime import timedelta + +import numpy as np +import xarray as xr +from parcels import Field, FieldSet + +from virtualship import Location, Spacetime +from virtualship.instruments.ctd_bgc import CTD_BGC, simulate_ctd_bgc + + +def test_simulate_ctd_bgcs(tmpdir) -> None: + # arbitrary time offset for the dummy fieldset + base_time = datetime.datetime.strptime("1950-01-01", "%Y-%m-%d") + + # where to cast CTD_BGCs + ctd_bgcs = [ + CTD_BGC( + spacetime=Spacetime( + location=Location(latitude=0, longitude=1), + time=base_time + datetime.timedelta(hours=0), + ), + min_depth=0, + max_depth=float("-inf"), + ), + CTD_BGC( + spacetime=Spacetime( + location=Location(latitude=1, longitude=0), + time=base_time, + ), + min_depth=0, + max_depth=float("-inf"), + ), + ] + + # expected observations for ctd_bgcs at surface and at maximum depth + ctd_bgc_exp = [ + { + "surface": { + "o2": 9, + "chl": 10, + "lat": ctd_bgcs[0].spacetime.location.lat, + "lon": ctd_bgcs[0].spacetime.location.lon, + }, + "maxdepth": { + "o2": 11, + "chl": 12, + "lat": ctd_bgcs[0].spacetime.location.lat, + "lon": ctd_bgcs[0].spacetime.location.lon, + }, + }, + { + "surface": { + "o2": 9, + "chl": 10, + "lat": ctd_bgcs[1].spacetime.location.lat, + "lon": ctd_bgcs[1].spacetime.location.lon, + }, + "maxdepth": { + "o2": 11, + "chl": 12, + "lat": ctd_bgcs[1].spacetime.location.lat, + "lon": ctd_bgcs[1].spacetime.location.lon, + }, + }, + ] + + # create fieldset based on the expected observations + # indices are time, depth, latitude, longitude + u = np.zeros((2, 2, 2, 2)) + v = np.zeros((2, 2, 2, 2)) + o2 = np.zeros((2, 2, 2, 2)) # o2 field + chl = np.zeros((2, 2, 2, 2)) # chl field + + o2[:, 1, 0, 1] = ctd_bgc_exp[0]["surface"]["o2"] + o2[:, 0, 0, 1] = ctd_bgc_exp[0]["maxdepth"]["o2"] + o2[:, 1, 1, 0] = ctd_bgc_exp[1]["surface"]["o2"] + o2[:, 0, 1, 0] = ctd_bgc_exp[1]["maxdepth"]["o2"] + + chl[:, 1, 0, 1] = ctd_bgc_exp[0]["surface"]["chl"] + chl[:, 0, 0, 1] = ctd_bgc_exp[0]["maxdepth"]["chl"] + chl[:, 1, 1, 0] = ctd_bgc_exp[1]["surface"]["chl"] + chl[:, 0, 1, 0] = ctd_bgc_exp[1]["maxdepth"]["chl"] + + fieldset = FieldSet.from_data( + {"V": v, "U": u, "o2": o2, "chl": chl}, + { + "time": [ + np.datetime64(base_time + datetime.timedelta(hours=0)), + np.datetime64(base_time + datetime.timedelta(hours=1)), + ], + "depth": [-1000, 0], + "lat": [0, 1], + "lon": [0, 1], + }, + ) + fieldset.add_field(Field("bathymetry", [-1000], lon=0, lat=0)) + + # perform simulation + out_path = tmpdir.join("out.zarr") + + simulate_ctd_bgc( + ctd_bgcs=ctd_bgcs, + fieldset=fieldset, + out_path=out_path, + outputdt=timedelta(seconds=10), + ) + + # test if output is as expected + results = xr.open_zarr(out_path) + + assert len(results.trajectory) == len(ctd_bgcs) + + for ctd_i, (traj, exp_bothloc) in enumerate( + zip(results.trajectory, ctd_bgc_exp, strict=True) + ): + obs_surface = results.sel(trajectory=traj, obs=0) + min_index = np.argmin(results.sel(trajectory=traj)["z"].data) + obs_maxdepth = results.sel(trajectory=traj, obs=min_index) + + for obs, loc in [ + (obs_surface, "surface"), + (obs_maxdepth, "maxdepth"), + ]: + exp = exp_bothloc[loc] + for var in ["o2", "chl", "lat", "lon"]: + obs_value = obs[var].values.item() + exp_value = exp[var] + assert np.isclose(obs_value, exp_value), ( + f"Observation incorrect {ctd_i=} {loc=} {var=} {obs_value=} {exp_value=}." + ) From d73401180660b7dd542113c9354547bda09b9cce Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 13 May 2025 09:53:01 +0200 Subject: [PATCH 14/15] new ctd_bgc instrument --- src/virtualship/instruments/ctd_bgc.py | 143 +++++++++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 src/virtualship/instruments/ctd_bgc.py diff --git a/src/virtualship/instruments/ctd_bgc.py b/src/virtualship/instruments/ctd_bgc.py new file mode 100644 index 00000000..cb218e3a --- /dev/null +++ b/src/virtualship/instruments/ctd_bgc.py @@ -0,0 +1,143 @@ +"""CTD_BGC instrument.""" + +from dataclasses import dataclass +from datetime import timedelta +from pathlib import Path + +import numpy as np +from parcels import FieldSet, JITParticle, ParticleSet, Variable + +from ..spacetime import Spacetime + + +@dataclass +class CTD_BGC: + """Configuration for a single BGC CTD.""" + + spacetime: Spacetime + min_depth: float + max_depth: float + + +_CTD_BGCParticle = JITParticle.add_variables( + [ + Variable("o2", dtype=np.float32, initial=np.nan), + Variable("chl", dtype=np.float32, initial=np.nan), + Variable("raising", dtype=np.int8, initial=0.0), # bool. 0 is False, 1 is True. + Variable("max_depth", dtype=np.float32), + Variable("min_depth", dtype=np.float32), + Variable("winch_speed", dtype=np.float32), + ] +) + + +def _sample_o2(particle, fieldset, time): + particle.o2 = fieldset.o2[time, particle.depth, particle.lat, particle.lon] + + +def _sample_chlorophyll(particle, fieldset, time): + particle.chl = fieldset.chl[time, particle.depth, particle.lat, particle.lon] + + +def _ctd_bgc_cast(particle, fieldset, time): + # lowering + if particle.raising == 0: + particle_ddepth = -particle.winch_speed * particle.dt + if particle.depth + particle_ddepth < particle.max_depth: + particle.raising = 1 + particle_ddepth = -particle_ddepth + # raising + else: + particle_ddepth = particle.winch_speed * particle.dt + if particle.depth + particle_ddepth > particle.min_depth: + particle.delete() + + +def simulate_ctd_bgc( + fieldset: FieldSet, + out_path: str | Path, + ctd_bgcs: list[CTD_BGC], + outputdt: timedelta, +) -> None: + """ + Use Parcels to simulate a set of BGC CTDs in a fieldset. + + :param fieldset: The fieldset to simulate the BGC CTDs in. + :param out_path: The path to write the results to. + :param ctds: A list of BGC CTDs to simulate. + :param outputdt: Interval which dictates the update frequency of file output during simulation + :raises ValueError: Whenever provided BGC CTDs, fieldset, are not compatible with this function. + """ + WINCH_SPEED = 1.0 # sink and rise speed in m/s + DT = 10.0 # dt of CTD simulation integrator + + if len(ctd_bgcs) == 0: + print( + "No BGC CTDs provided. Parcels currently crashes when providing an empty particle set, so no BGC CTD simulation will be done and no files will be created." + ) + # TODO when Parcels supports it this check can be removed. + return + + fieldset_starttime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[0]) + fieldset_endtime = fieldset.time_origin.fulltime(fieldset.U.grid.time_full[-1]) + + # deploy time for all ctds should be later than fieldset start time + if not all( + [ + np.datetime64(ctd_bgc.spacetime.time) >= fieldset_starttime + for ctd_bgc in ctd_bgcs + ] + ): + raise ValueError("BGC CTD deployed before fieldset starts.") + + # depth the bgc ctd will go to. shallowest between bgc ctd max depth and bathymetry. + max_depths = [ + max( + ctd_bgc.max_depth, + fieldset.bathymetry.eval( + z=0, + y=ctd_bgc.spacetime.location.lat, + x=ctd_bgc.spacetime.location.lon, + time=0, + ), + ) + for ctd_bgc in ctd_bgcs + ] + + # CTD depth can not be too shallow, because kernel would break. + # This shallow is not useful anyway, no need to support. + if not all([max_depth <= -DT * WINCH_SPEED for max_depth in max_depths]): + raise ValueError( + f"BGC CTD max_depth or bathymetry shallower than maximum {-DT * WINCH_SPEED}" + ) + + # define parcel particles + ctd_bgc_particleset = ParticleSet( + fieldset=fieldset, + pclass=_CTD_BGCParticle, + lon=[ctd_bgc.spacetime.location.lon for ctd_bgc in ctd_bgcs], + lat=[ctd_bgc.spacetime.location.lat for ctd_bgc in ctd_bgcs], + depth=[ctd_bgc.min_depth for ctd_bgc in ctd_bgcs], + time=[ctd_bgc.spacetime.time for ctd_bgc in ctd_bgcs], + max_depth=max_depths, + min_depth=[ctd_bgc.min_depth for ctd_bgc in ctd_bgcs], + winch_speed=[WINCH_SPEED for _ in ctd_bgcs], + ) + + # define output file for the simulation + out_file = ctd_bgc_particleset.ParticleFile(name=out_path, outputdt=outputdt) + + # execute simulation + ctd_bgc_particleset.execute( + [_sample_o2, _sample_chlorophyll, _ctd_bgc_cast], + endtime=fieldset_endtime, + dt=DT, + verbose_progress=False, + output_file=out_file, + ) + + # there should be no particles left, as they delete themselves when they resurface + if len(ctd_bgc_particleset.particledata) != 0: + raise ValueError( + "Simulation ended before BGC CTD resurfaced. This most likely means the field time dimension did not match the simulation time span." + ) From 09e9d872cfa11706787f95984a617d967dcec25c Mon Sep 17 00:00:00 2001 From: j-atkins <106238905+j-atkins@users.noreply.github.com> Date: Tue, 13 May 2025 09:59:33 +0200 Subject: [PATCH 15/15] add ctd_bgc sampling capability to virtualship run execution --- src/virtualship/expedition/__init__.py | 2 + src/virtualship/expedition/do_expedition.py | 1 + src/virtualship/expedition/input_data.py | 50 +++++++++++++++++++ .../expedition/simulate_measurements.py | 14 ++++++ .../expedition/simulate_schedule.py | 12 +++++ 5 files changed, 79 insertions(+) diff --git a/src/virtualship/expedition/__init__.py b/src/virtualship/expedition/__init__.py index 74403732..65b7cf11 100644 --- a/src/virtualship/expedition/__init__.py +++ b/src/virtualship/expedition/__init__.py @@ -6,6 +6,7 @@ from .ship_config import ( ADCPConfig, ArgoFloatConfig, + CTD_BGCConfig, CTDConfig, DrifterConfig, ShipConfig, @@ -17,6 +18,7 @@ "ADCPConfig", "ArgoFloatConfig", "CTDConfig", + "CTD_BGCConfig", "DrifterConfig", "InputData", "InstrumentType", diff --git a/src/virtualship/expedition/do_expedition.py b/src/virtualship/expedition/do_expedition.py index 97a66f7f..a4243361 100644 --- a/src/virtualship/expedition/do_expedition.py +++ b/src/virtualship/expedition/do_expedition.py @@ -136,6 +136,7 @@ def _load_input_data( load_adcp=ship_config.adcp_config is not None, load_argo_float=ship_config.argo_float_config is not None, load_ctd=ship_config.ctd_config is not None, + load_ctd_bgc=ship_config.ctd_bgc_config is not None, load_drifter=ship_config.drifter_config is not None, load_xbt=ship_config.xbt_config is not None, load_ship_underwater_st=ship_config.ship_underwater_st_config is not None, diff --git a/src/virtualship/expedition/input_data.py b/src/virtualship/expedition/input_data.py index 554efb78..ca99f7e8 100644 --- a/src/virtualship/expedition/input_data.py +++ b/src/virtualship/expedition/input_data.py @@ -15,6 +15,7 @@ class InputData: adcp_fieldset: FieldSet | None argo_float_fieldset: FieldSet | None ctd_fieldset: FieldSet | None + ctd_bgc_fieldset: FieldSet | None drifter_fieldset: FieldSet | None xbt_fieldset: FieldSet | None ship_underwater_st_fieldset: FieldSet | None @@ -26,6 +27,7 @@ def load( load_adcp: bool, load_argo_float: bool, load_ctd: bool, + load_ctd_bgc: bool, load_drifter: bool, load_xbt: bool, load_ship_underwater_st: bool, @@ -39,6 +41,7 @@ def load( :param load_adcp: Whether to load the ADCP fieldset. :param load_argo_float: Whether to load the argo float fieldset. :param load_ctd: Whether to load the CTD fieldset. + :param load_ctd_bgc: Whether to load the CTD BGC fieldset. :param load_drifter: Whether to load the drifter fieldset. :param load_ship_underwater_st: Whether to load the ship underwater ST fieldset. :returns: An instance of this class with loaded fieldsets. @@ -51,6 +54,10 @@ def load( argo_float_fieldset = cls._load_argo_float_fieldset(directory) else: argo_float_fieldset = None + if load_ctd_bgc: + ctd_bgc_fieldset = cls._load_ctd_bgc_fieldset(directory) + else: + ctd_bgc_fieldset = None if load_adcp or load_ctd or load_ship_underwater_st or load_xbt: ship_fieldset = cls._load_ship_fieldset(directory) if load_adcp: @@ -74,6 +81,7 @@ def load( adcp_fieldset=adcp_fieldset, argo_float_fieldset=argo_float_fieldset, ctd_fieldset=ctd_fieldset, + ctd_bgc_fieldset=ctd_bgc_fieldset, drifter_fieldset=drifter_fieldset, xbt_fieldset=xbt_fieldset, ship_underwater_st_fieldset=ship_underwater_st_fieldset, @@ -122,6 +130,48 @@ def _load_ship_fieldset(cls, directory: Path) -> FieldSet: return fieldset + @classmethod + def _load_ctd_bgc_fieldset(cls, directory: Path) -> FieldSet: + filenames = { + "U": directory.joinpath("ship_uv.nc"), + "V": directory.joinpath("ship_uv.nc"), + "o2": directory.joinpath("ctd_bgc_o2.nc"), + "chl": directory.joinpath("ctd_bgc_chloro.nc"), + } + variables = {"U": "uo", "V": "vo", "o2": "o2", "chl": "chl"} + dimensions = { + "lon": "longitude", + "lat": "latitude", + "time": "time", + "depth": "depth", + } + + fieldset = FieldSet.from_netcdf( + filenames, variables, dimensions, allow_time_extrapolation=True + ) + fieldset.o2.interp_method = "linear_invdist_land_tracer" + fieldset.chl.interp_method = "linear_invdist_land_tracer" + + # make depth negative + for g in fieldset.gridset.grids: + g.negate_depth() + + # add bathymetry data + bathymetry_file = directory.joinpath("bathymetry.nc") + bathymetry_variables = ("bathymetry", "deptho") + bathymetry_dimensions = {"lon": "longitude", "lat": "latitude"} + bathymetry_field = Field.from_netcdf( + bathymetry_file, bathymetry_variables, bathymetry_dimensions + ) + # make depth negative + bathymetry_field.data = -bathymetry_field.data + fieldset.add_field(bathymetry_field) + + # read in data already + fieldset.computeTimeChunk(0, 1) + + return fieldset + @classmethod def _load_drifter_fieldset(cls, directory: Path) -> FieldSet: filenames = { diff --git a/src/virtualship/expedition/simulate_measurements.py b/src/virtualship/expedition/simulate_measurements.py index bf28c989..c7cd8746 100644 --- a/src/virtualship/expedition/simulate_measurements.py +++ b/src/virtualship/expedition/simulate_measurements.py @@ -6,6 +6,7 @@ from ..instruments.adcp import simulate_adcp from ..instruments.argo_float import simulate_argo_floats from ..instruments.ctd import simulate_ctd +from ..instruments.ctd_bgc import simulate_ctd_bgc from ..instruments.drifter import simulate_drifters from ..instruments.ship_underwater_st import simulate_ship_underwater_st from ..instruments.xbt import simulate_xbt @@ -75,6 +76,19 @@ def simulate_measurements( outputdt=timedelta(seconds=10), ) + if len(measurements.ctd_bgcs) > 0: + print("Simulating BGC CTD casts.") + if ship_config.ctd_bgc_config is None: + raise RuntimeError("No configuration for CTD_BGC provided.") + if input_data.ctd_bgc_fieldset is None: + raise RuntimeError("No fieldset for CTD_BGC provided.") + simulate_ctd_bgc( + out_path=expedition_dir.joinpath("results", "ctd_bgc.zarr"), + fieldset=input_data.ctd_bgc_fieldset, + ctd_bgcs=measurements.ctd_bgcs, + outputdt=timedelta(seconds=10), + ) + if len(measurements.drifters) > 0: print("Simulating drifters") if ship_config.drifter_config is None: diff --git a/src/virtualship/expedition/simulate_schedule.py b/src/virtualship/expedition/simulate_schedule.py index df36b8b2..596d16a2 100644 --- a/src/virtualship/expedition/simulate_schedule.py +++ b/src/virtualship/expedition/simulate_schedule.py @@ -9,6 +9,7 @@ from ..instruments.argo_float import ArgoFloat from ..instruments.ctd import CTD +from ..instruments.ctd_bgc import CTD_BGC from ..instruments.drifter import Drifter from ..instruments.xbt import XBT from ..location import Location @@ -42,6 +43,7 @@ class MeasurementsToSimulate: argo_floats: list[ArgoFloat] = field(default_factory=list, init=False) drifters: list[Drifter] = field(default_factory=list, init=False) ctds: list[CTD] = field(default_factory=list, init=False) + ctd_bgcs: list[CTD_BGC] = field(default_factory=list, init=False) xbts: list[XBT] = field(default_factory=list, init=False) @@ -102,6 +104,7 @@ def simulate(self) -> ScheduleOk | ScheduleProblem: # check if waypoint was reached in time if waypoint.time is not None and self._time > waypoint.time: print( + # TODO: I think this should be wp_i + 1, not wp_i; otherwise it will be off by one f"Waypoint {wp_i} could not be reached in time. Current time: {self._time}. Waypoint time: {waypoint.time}." ) return ScheduleProblem(self._time, wp_i) @@ -251,6 +254,15 @@ def _make_measurements(self, waypoint: Waypoint) -> timedelta: ) ) time_costs.append(self._ship_config.ctd_config.stationkeeping_time) + elif instrument is InstrumentType.CTD_BGC: + self._measurements_to_simulate.ctd_bgcs.append( + CTD_BGC( + spacetime=Spacetime(self._location, self._time), + min_depth=self._ship_config.ctd_bgc_config.min_depth_meter, + max_depth=self._ship_config.ctd_bgc_config.max_depth_meter, + ) + ) + time_costs.append(self._ship_config.ctd_bgc_config.stationkeeping_time) elif instrument is InstrumentType.DRIFTER: self._measurements_to_simulate.drifters.append( Drifter(