Merge pull request #703 from simvue-io/wk9874/offline_fixes

General fixes for v2.0.0a1

Author: wk9874

poetry.lock

@@ -46,7 +46,7 @@ dependencies = [
   "humanfriendly (>=10.0,<11.0)",
   "randomname (>=0.2.1,<0.3.0)",
   "codecarbon (>=2.8.3,<3.0.0)",
-  "numpy (>=2.2.2,<3.0.0)",
+  "numpy (>=2.0.0,<3.0.0)",
   "flatdict (>=4.0.1,<5.0.0)",
   "semver (>=3.0.4,<4.0.0)",
   "email-validator (>=2.2.0,<3.0.0)",

pyproject.toml

@@ -23,6 +23,62 @@ def __new__(cls, identifier: str | None = None, **kwargs):
         else:
             return ObjectArtifact(identifier=identifier, **kwargs)
 
+    @classmethod
+    def from_run(
+        cls,
+        run_id: str,
+        category: typing.Literal["input", "output", "code"] | None = None,
+        **kwargs,
+    ) -> typing.Generator[tuple[str, FileArtifact | ObjectArtifact], None, None]:
+        """Return artifacts associated with a given run.
+
+        Parameters
+        ----------
+        run_id : str
+            The ID of the run to retriece artifacts from
+        category : typing.Literal["input", "output", "code"] | None, optional
+            The category of artifacts to return, by default all artifacts are returned
+
+        Returns
+        -------
+        typing.Generator[tuple[str, FileArtifact | ObjectArtifact], None, None]
+            The artifacts
+
+        Yields
+        ------
+        Iterator[typing.Generator[tuple[str, FileArtifact | ObjectArtifact], None, None]]
+            identifier for artifact
+            the artifact itself as a class instance
+
+        Raises
+        ------
+        ObjectNotFoundError
+            Raised if artifacts could not be found for that run
+        """
+        _temp = ArtifactBase(**kwargs)
+        _url = URL(_temp._user_config.server.url) / f"runs/{run_id}/artifacts"
+        _response = sv_get(
+            url=f"{_url}", params={"category": category}, headers=_temp._headers
+        )
+        _json_response = get_json_from_response(
+            expected_type=list,
+            response=_response,
+            expected_status=[http.HTTPStatus.OK, http.HTTPStatus.NOT_FOUND],
+            scenario=f"Retrieval of artifacts for run '{run_id}'",
+        )
+
+        if _response.status_code == http.HTTPStatus.NOT_FOUND or not _json_response:
+            raise ObjectNotFoundError(
+                _temp._label, category, extra=f"for run '{run_id}'"
+            )
+
+        for _entry in _json_response:
+            _id = _entry.pop("id")
+            yield (
+                _id,
+                Artifact(_local=True, _read_only=True, identifier=_id, **_entry),
+            )
+
     @classmethod
     def from_name(
         cls, run_id: str, name: str, **kwargs
@@ -99,21 +155,9 @@ def get(
         if (_data := _json_response.get("data")) is None:
             raise RuntimeError(f"Expected key 'data' for retrieval of {_label}s")
 
-        _out_dict: dict[str, FileArtifact | ObjectArtifact] = {}
-
         for _entry in _data:
             _id = _entry.pop("id")
-            if _entry["original_path"]:
-                yield (
-                    _id,
-                    FileArtifact(
-                        _local=True, _read_only=True, identifier=_id, **_entry
-                    ),
-                )
-            else:
-                yield (
-                    _id,
-                    ObjectArtifact(
-                        _local=True, _read_only=True, identifier=_id, **_entry
-                    ),
-                )
+            yield (
+                _id,
+                Artifact(_local=True, _read_only=True, identifier=_id, **_entry),
+            )

simvue/api/objects/artifact/fetch.py

@@ -52,14 +52,15 @@ def new(
 
         if _mime_type not in get_mimetypes():
             raise ValueError(f"Invalid MIME type '{mime_type}' specified")
-        file_path = pathlib.Path(file_path)
-        _file_size = file_path.stat().st_size
-        _file_orig_path = file_path.expanduser().absolute()
-        _file_checksum = calculate_sha256(f"{file_path}", is_file=True)
 
-        kwargs.pop("original_path", None)
-        kwargs.pop("size", None)
-        kwargs.pop("checksum", None)
+        if _file_orig_path := kwargs.pop("original_path", None):
+            _file_size = kwargs.pop("size")
+            _file_checksum = kwargs.pop("checksum")
+        else:
+            file_path = pathlib.Path(file_path)
+            _file_size = file_path.stat().st_size
+            _file_orig_path = file_path.expanduser().absolute()
+            _file_checksum = calculate_sha256(f"{file_path}", is_file=True)
 
         _artifact = FileArtifact(
             name=name,

simvue/api/objects/artifact/file.py

@@ -11,6 +11,7 @@
 import typing
 import pydantic
 import datetime
+import time
 
 try:
     from typing import Self
@@ -257,6 +258,19 @@ def created(self) -> datetime.datetime | None:
             datetime.datetime.strptime(_created, DATETIME_FORMAT) if _created else None
         )
 
+    @created.setter
+    @write_only
+    @pydantic.validate_call
+    def created(self, created: datetime.datetime) -> None:
+        self._staging["created"] = created.strftime(DATETIME_FORMAT)
+
+    @property
+    @staging_check
+    def runtime(self) -> datetime.datetime | None:
+        """Retrieve created datetime for the run"""
+        _runtime: str | None = self._get_attribute("runtime")
+        return time.strptime(_runtime, "%H:%M:%S.%f") if _runtime else None
+
     @property
     @staging_check
     def started(self) -> datetime.datetime | None:

simvue/api/objects/run.py

@@ -9,7 +9,6 @@
 import contextlib
 import json
 import logging
-import os
 import pathlib
 import typing
 import http
@@ -45,12 +44,9 @@
 def _download_artifact_to_file(
     artifact: Artifact, output_dir: pathlib.Path | None
 ) -> None:
-    try:
-        _file_name = os.path.basename(artifact.name)
-    except AttributeError:
-        _file_name = os.path.basename(artifact)
-    _output_file = (output_dir or pathlib.Path.cwd()).joinpath(_file_name)
-
+    _output_file = (output_dir or pathlib.Path.cwd()).joinpath(artifact.name)
+    # If this is a hierarchical structure being downloaded, need to create directories
+    _output_file.parent.mkdir(parents=True, exist_ok=True)
     with _output_file.open("wb") as out_f:
         for content in artifact.download_content():
             out_f.write(content)
@@ -565,34 +561,27 @@ def get_artifacts_as_files(
         run_id: str,
         category: typing.Literal["input", "output", "code"] | None = None,
         output_dir: pydantic.DirectoryPath | None = None,
-        startswith: str | None = None,
-        contains: str | None = None,
-        endswith: str | None = None,
     ) -> None:
         """Retrieve artifacts from the given run as a set of files
 
         Parameters
         ----------
         run_id : str
             the unique identifier for the run
+        category : typing.Literal["input", "output", "code"] |
+            the type of files to retrieve
         output_dir : str | None, optional
             location to download files to, the default of None will download
             them to the current working directory
-        startswith : str, optional
-            only download artifacts with this prefix in their name, by default None
-        contains : str, optional
-            only download artifacts containing this term in their name, by default None
-        endswith : str, optional
-            only download artifacts ending in this term in their name, by default None
 
         Raises
         ------
         RuntimeError
             if there was a failure retrieving artifacts from the server
         """
-        _artifacts: typing.Generator[tuple[str, Artifact], None, None] = Artifact.get(
-            runs=json.dumps([run_id]), category=category
-        )  # type: ignore
+        _artifacts: typing.Generator[tuple[str, Artifact], None, None] = (
+            Artifact.from_run(run_id=run_id, category=category)
+        )
 
         with ThreadPoolExecutor(CONCURRENT_DOWNLOADS) as executor:
             futures = [

simvue/client.py

@@ -21,7 +21,6 @@
 QUEUE_SIZE = 10000
 
 logger = logging.getLogger(__name__)
-logger.setLevel(logging.DEBUG)
 
 
 class QueuedDispatcher(threading.Thread, DispatcherBaseClass):

simvue/factory/dispatch/queued.py

@@ -25,7 +25,7 @@
 import typing
 import warnings
 import uuid
-
+import randomname
 import click
 import psutil
 
@@ -252,19 +252,12 @@ def _handle_exception_throw(
             else f"An exception was thrown: {_exception_thrown}"
         )
 
-        self.log_event(_event_msg)
-        self.set_status("terminated" if _is_terminated else "failed")
-
         # If the dispatcher has already been aborted then this will
         # fail so just continue without the event
         with contextlib.suppress(RuntimeError):
-            self.log_event(f"{_exception_thrown}: {value}")
+            self.log_event(_event_msg)
 
-        if not traceback:
-            return
-
-        with contextlib.suppress(RuntimeError):
-            self.log_event(f"Traceback: {traceback}")
+        self.set_status("terminated" if _is_terminated else "failed")
 
     def __exit__(
         self,
@@ -470,12 +463,13 @@ def _start(self, reconnect: bool = False) -> bool:
 
         logger.debug("Starting run")
 
+        self._start_time = time.time()
+
         if self._sv_obj and self._sv_obj.status != "running":
             self._sv_obj.status = self._status
+            self._sv_obj.started = self._start_time
             self._sv_obj.commit()
 
-        self._start_time = time.time()
-
         if self._pid == 0:
             self._pid = os.getpid()
 
@@ -655,6 +649,8 @@ def init(
         if name and not re.match(r"^[a-zA-Z0-9\-\_\s\/\.:]+$", name):
             self._error("specified name is invalid")
             return False
+        elif not name and self._user_config.run.mode == "offline":
+            name = randomname.get_name()
 
         self._name = name
 
@@ -695,6 +691,7 @@ def init(
         self._sv_obj.metadata = (metadata or {}) | git_info(os.getcwd()) | environment()
         self._sv_obj.heartbeat_timeout = timeout
         self._sv_obj.alerts = []
+        self._sv_obj.created = time.time()
         self._sv_obj.notifications = notification
 
         if self._status == "running":
@@ -724,7 +721,7 @@ def init(
                 fg="green" if self._term_color else None,
             )
             click.secho(
-                f"[simvue] Monitor in the UI at {self._user_config.server.url}/dashboard/runs/run/{self._id}",
+                f"[simvue] Monitor in the UI at {self._user_config.server.url.rsplit('/api', 1)[0]}/dashboard/runs/run/{self._id}",
                 bold=self._term_color,
                 fg="green" if self._term_color else None,
             )
@@ -1469,7 +1466,7 @@ def set_status(
     ) -> bool:
         """Set run status
 
-        status to assign to this run
+        status to assign to this run once finished
 
         Parameters
         ----------
@@ -1489,6 +1486,7 @@ def set_status(
 
         if self._sv_obj:
             self._sv_obj.status = status
+            self._sv_obj.endtime = time.time()
             self._sv_obj.commit()
             return True
 
@@ -1641,9 +1639,7 @@ def add_alerts(
         if names and not ids:
             try:
                 if alerts := Alert.get(offline=self._user_config.run.mode == "offline"):
-                    for alert in alerts:
-                        if alert[1].name in names:
-                            ids.append(alert[1].id)
+                    ids += [id for id, alert in alerts if alert.name in names]
                 else:
                     self._error("No existing alerts")
                     return False

simvue/run.py

@@ -358,7 +358,7 @@ def validate_timestamp(timestamp):
     Validate a user-provided timestamp
     """
     try:
-        datetime.datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S.%f")
+        datetime.datetime.strptime(timestamp, DATETIME_FORMAT)
     except ValueError:
         return False
 

simvue/utilities.py

@@ -145,12 +145,21 @@ def test_get_artifacts_as_files(
             create_test_run[1]["run_id"], category=category, output_dir=tempd
         )
         files = [os.path.basename(i) for i in glob.glob(os.path.join(tempd, "*"))]
-        if not category or category == "input":
-            assert create_test_run[1]["file_1"] in files
-        if not category or category == "output":
-            assert create_test_run[1]["file_2"] in files
-        if not category or category == "code":
-            assert create_test_run[1]["file_3"] in files
+        
+        if not category:
+            expected_files = ["file_1", "file_2", "file_3"]
+        elif category == "input":
+            expected_files = ["file_1"]
+        elif category == "output":
+            expected_files = ["file_2"]
+        elif category == "code":
+            expected_files = ["file_3"]
+            
+        for file in ["file_1", "file_2", "file_3"]:
+            if file in expected_files:
+                assert create_test_run[1][file] in files
+            else:
+                assert create_test_run[1][file] not in files
 
 
 @pytest.mark.dependency