Merge pull request #714 from simvue-io/wk9874/2.0.0_a2

Bug Fixes for v2.0.0a2

Author: wk9874

simvue/api/objects/base.py

@@ -50,6 +50,8 @@ def _wrapper(self) -> typing.Any:
             raise RuntimeError(
                 f"Cannot use 'staging_check' decorator on type '{type(self).__name__}'"
             )
+        if _sv_obj._offline:
+            return member_func(self)
         if not _sv_obj._read_only and member_func.__name__ in _sv_obj._staging:
             _sv_obj._logger.warning(
                 f"Uncommitted change found for attribute '{member_func.__name__}'"

simvue/api/objects/run.py

@@ -315,7 +315,14 @@ def events(
 
     @write_only
     def send_heartbeat(self) -> dict[str, typing.Any] | None:
-        if self._offline or not self._identifier:
+        if not self._identifier:
+            return None
+
+        if self._offline:
+            if not (_dir := self._local_staging_file.parent).exists():
+                _dir.mkdir(parents=True)
+            _heartbeat_file = self._local_staging_file.with_suffix(".heartbeat")
+            _heartbeat_file.touch()
             return None
 
         _url = self._base_url

simvue/config/user.py

@@ -180,7 +180,7 @@ def fetch(
         except FileNotFoundError:
             if not server_token or not server_url:
                 _config_dict = {"server": {}}
-                logger.warning("No config file found, checking environment variables")
+                logger.debug("No config file found, checking environment variables")
 
         _config_dict["server"] = _config_dict.get("server", {})
 

simvue/metadata.py

@@ -12,7 +12,6 @@
 import toml
 import logging
 import pathlib
-import flatdict
 
 from simvue.utilities import simvue_timestamp
 
@@ -64,7 +63,7 @@ def git_info(repository: str) -> dict[str, typing.Any]:
         )
         return {
             "git": {
-                "authors": json.dumps(list(author_list)),
+                "authors": list(author_list),
                 "ref": ref,
                 "msg": current_commit.message.strip(),
                 "time_stamp": simvue_timestamp(current_commit.committed_datetime),
@@ -84,34 +83,32 @@ def _python_env(repository: pathlib.Path) -> dict[str, typing.Any]:
     if (pyproject_file := pathlib.Path(repository).joinpath("pyproject.toml")).exists():
         content = toml.load(pyproject_file)
         if (poetry_content := content.get("tool", {}).get("poetry", {})).get("name"):
-            python_meta |= {
-                "python.project.name": poetry_content["name"],
-                "python.project.version": poetry_content["version"],
+            python_meta["project"] = {
+                "name": poetry_content["name"],
+                "version": poetry_content["version"],
             }
         elif other_content := content.get("project"):
-            python_meta |= {
-                "python.project.name": other_content["name"],
-                "python.project.version": other_content["version"],
+            python_meta["project"] = {
+                "name": other_content["name"],
+                "version": other_content["version"],
             }
 
     if (poetry_lock_file := pathlib.Path(repository).joinpath("poetry.lock")).exists():
         content = toml.load(poetry_lock_file).get("package", {})
-        python_meta |= {
-            f"python.environment.{package['name']}": package["version"]
-            for package in content
+        python_meta["environment"] = {
+            package["name"]: package["version"] for package in content
         }
     elif (uv_lock_file := pathlib.Path(repository).joinpath("uv.lock")).exists():
         content = toml.load(uv_lock_file).get("package", {})
-        python_meta |= {
-            f"python.environment.{package['name']}": package["version"]
-            for package in content
+        python_meta["environment"] = {
+            package["name"]: package["version"] for package in content
         }
     else:
         with contextlib.suppress((KeyError, ImportError)):
             from pip._internal.operations.freeze import freeze
 
-            python_meta |= {
-                f"python.environment.{entry[0]}": entry[-1]
+            python_meta["environment"] = {
+                entry[0]: entry[-1]
                 for line in freeze(local_only=True)
                 if (entry := line.split("=="))
             }
@@ -126,35 +123,33 @@ def _rust_env(repository: pathlib.Path) -> dict[str, typing.Any]:
     if (cargo_file := pathlib.Path(repository).joinpath("Cargo.toml")).exists():
         content = toml.load(cargo_file).get("package", {})
         if version := content.get("version"):
-            rust_meta |= {"rust.project.version": version}
+            rust_meta.setdefault("project", {})["version"] = version
 
         if name := content.get("name"):
-            rust_meta |= {"rust.project.name": name}
+            rust_meta.setdefault("project", {})["name"] = name
 
     if not (cargo_lock := pathlib.Path(repository).joinpath("Cargo.lock")).exists():
-        return {}
+        return rust_meta
 
     cargo_dat = toml.load(cargo_lock)
-
-    return rust_meta | {
-        f"rust.environment.{dependency['name']}": dependency["version"]
+    rust_meta["environment"] = {
+        dependency["name"]: dependency["version"]
         for dependency in cargo_dat.get("package")
     }
 
+    return rust_meta
+
 
 def _julia_env(repository: pathlib.Path) -> dict[str, typing.Any]:
     """Retrieve a dictionary of Julia dependencies if a project file is available"""
     julia_meta: dict[str, str] = {}
     if (project_file := pathlib.Path(repository).joinpath("Project.toml")).exists():
         content = toml.load(project_file)
-        julia_meta |= {
-            f"julia.project.{key}": value
-            for key, value in content.items()
-            if not isinstance(value, dict)
+        julia_meta["project"] = {
+            key: value for key, value in content.items() if not isinstance(value, dict)
         }
-        julia_meta |= {
-            f"julia.environment.{key}": value
-            for key, value in content.get("compat", {}).items()
+        julia_meta["environment"] = {
+            key: value for key, value in content.get("compat", {}).items()
         }
     return julia_meta
 
@@ -171,13 +166,11 @@ def _node_js_env(repository: pathlib.Path) -> dict[str, typing.Any]:
             )
             return {}
 
-        js_meta |= {
-            f"javascript.project.{key}": value
-            for key, value in content.items()
-            if key in ("name", "version")
+        js_meta["project"] = {
+            key: value for key, value in content.items() if key in ("name", "version")
         }
-        js_meta |= {
-            f"javascript.environment.{key.replace('@', '')}": value["version"]
+        js_meta["environment"] = {
+            key.replace("@", ""): value["version"]
             for key, value in content.get(
                 "packages" if lfv in (2, 3) else "dependencies", {}
             ).items()
@@ -188,16 +181,13 @@ def _node_js_env(repository: pathlib.Path) -> dict[str, typing.Any]:
 
 def environment(repository: pathlib.Path = pathlib.Path.cwd()) -> dict[str, typing.Any]:
     """Retrieve environment metadata"""
-    _environment_meta = flatdict.FlatDict(
-        _python_env(repository), delimiter="."
-    ).as_dict()
-    _environment_meta |= flatdict.FlatDict(
-        _rust_env(repository), delimiter="."
-    ).as_dict()
-    _environment_meta |= flatdict.FlatDict(
-        _julia_env(repository), delimiter="."
-    ).as_dict()
-    _environment_meta |= flatdict.FlatDict(
-        _node_js_env(repository), delimiter="."
-    ).as_dict()
+    _environment_meta = {}
+    if _python_meta := _python_env(repository):
+        _environment_meta["python"] = _python_meta
+    if _rust_meta := _rust_env(repository):
+        _environment_meta["rust"] = _rust_meta
+    if _julia_meta := _julia_env(repository):
+        _environment_meta["julia"] = _julia_meta
+    if _js_meta := _node_js_env(repository):
+        _environment_meta["javascript"] = _js_meta
     return _environment_meta

simvue/sender.py

@@ -10,9 +10,11 @@
 import logging
 from concurrent.futures import ThreadPoolExecutor
 import threading
+import requests
 from simvue.config.user import SimvueConfiguration
 
 import simvue.api.objects
+from simvue.version import __version__
 
 UPLOAD_ORDER: list[str] = [
     "tenants",
@@ -134,7 +136,8 @@ def sender(
     objects_to_upload : list[str]
         Types of objects to upload, by default uploads all types of objects present in cache
     """
-    cache_dir = cache_dir or SimvueConfiguration.fetch().offline.cache
+    _user_config = SimvueConfiguration.fetch()
+    cache_dir = cache_dir or _user_config.offline.cache
     cache_dir.joinpath("server_ids").mkdir(parents=True, exist_ok=True)
     _id_mapping: dict[str, str] = {
         file_path.name.split(".")[0]: file_path.read_text()
@@ -160,4 +163,29 @@ def sender(
                     ),
                     _offline_files,
                 )
+
+    # Send heartbeats
+    _headers: dict[str, str] = {
+        "Authorization": f"Bearer {_user_config.server.token.get_secret_value()}",
+        "User-Agent": f"Simvue Python client {__version__}",
+    }
+
+    for _heartbeat_file in cache_dir.glob("runs/*.heartbeat"):
+        _offline_id = _heartbeat_file.name.split(".")[0]
+        _online_id = _id_mapping.get(_offline_id)
+        if not _online_id:
+            # Run has been closed - can just remove heartbeat and continue
+            _heartbeat_file.unlink()
+            continue
+        _logger.info(f"Sending heartbeat to run {_online_id}")
+        _response = requests.put(
+            f"{_user_config.server.url}/runs/{_online_id}/heartbeat",
+            headers=_headers,
+        )
+        if _response.status_code == 200:
+            _heartbeat_file.unlink()
+        else:
+            _logger.warning(
+                f"Attempting to send heartbeat to run {_online_id} returned status code {_response.status_code}."
+            )
     return _id_mapping

tests/unit/test_metadata.py

@@ -8,8 +8,8 @@
 @pytest.mark.local
 def test_cargo_env() -> None:
     metadata = sv_meta._rust_env(pathlib.Path(__file__).parents[1].joinpath("example_data"))
-    assert metadata["rust.environment.serde"] == "1.0.123"
-    assert metadata["rust.project.name"] == "example_project"
+    assert metadata["environment"]["serde"] == "1.0.123"
+    assert metadata["project"]["name"] == "example_project"
 
 @pytest.mark.metadata
 @pytest.mark.local
@@ -19,29 +19,36 @@ def test_cargo_env() -> None:
 def test_python_env(backend: str | None) -> None:
     if backend == "poetry":
         metadata = sv_meta._python_env(pathlib.Path(__file__).parents[1].joinpath("example_data", "python_poetry"))
-        assert metadata["python.project.name"] == "example-repo"
+        assert metadata["project"]["name"] == "example-repo"
     elif backend == "uv":
         metadata = sv_meta._python_env(pathlib.Path(__file__).parents[1].joinpath("example_data", "python_uv"))
-        assert metadata["python.project.name"] == "example-repo"
+        assert metadata["project"]["name"] == "example-repo"
     else:
         metadata = sv_meta._python_env(pathlib.Path(__file__).parents[1].joinpath("example_data"))
 
-    assert re.findall(r"\d+\.\d+\.\d+", metadata["python.environment.numpy"])
+    assert re.findall(r"\d+\.\d+\.\d+", metadata["environment"]["numpy"])
 
 
 @pytest.mark.metadata
 @pytest.mark.local
 def test_julia_env() -> None:
     metadata = sv_meta._julia_env(pathlib.Path(__file__).parents[1].joinpath("example_data"))
-    assert metadata["julia.project.name"] == "Julia Demo Project"
-    assert re.findall(r"\d+\.\d+\.\d+", metadata["julia.environment.AbstractDifferentiation"])
+    assert metadata["project"]["name"] == "Julia Demo Project"
+    assert re.findall(r"\d+\.\d+\.\d+", metadata["environment"]["AbstractDifferentiation"])
 
 
 @pytest.mark.metadata
 @pytest.mark.local
 def test_js_env() -> None:
     metadata = sv_meta._node_js_env(pathlib.Path(__file__).parents[1].joinpath("example_data"))
-    assert metadata["javascript.project.name"] == "my-awesome-project"
-    assert re.findall(r"\d+\.\d+\.\d+", metadata["javascript.environment.node_modules/dotenv"])
-
+    assert metadata["project"]["name"] == "my-awesome-project"
+    assert re.findall(r"\d+\.\d+\.\d+", metadata["environment"]["node_modules/dotenv"])
 
+@pytest.mark.metadata
+@pytest.mark.local
+def test_environment() -> None:
+    metadata = sv_meta.environment(pathlib.Path(__file__).parents[1].joinpath("example_data"))
+    assert metadata["python"]["project"]["name"] == "example-repo"
+    assert metadata["rust"]["project"]["name"] == "example_project"
+    assert metadata["julia"]["project"]["name"] == "Julia Demo Project"
+    assert metadata["javascript"]["project"]["name"] == "my-awesome-project"