diff --git a/simvue/client.py b/simvue/client.py index 9710517e..3c86abce 100644 --- a/simvue/client.py +++ b/simvue/client.py @@ -26,7 +26,7 @@ from .simvue_types import DeserializedContent from .utilities import check_extra, prettify_pydantic from .models import FOLDER_REGEX, NAME_REGEX -from .config import SimvueConfiguration +from .config.user import SimvueConfiguration if typing.TYPE_CHECKING: pass diff --git a/simvue/config/__init__.py b/simvue/config/__init__.py index db386d99..28232a84 100644 --- a/simvue/config/__init__.py +++ b/simvue/config/__init__.py @@ -5,5 +5,3 @@ This module contains definitions for the Simvue configuration options """ - -from .user import SimvueConfiguration as SimvueConfiguration diff --git a/simvue/config/files.py b/simvue/config/files.py new file mode 100644 index 00000000..baa80c2f --- /dev/null +++ b/simvue/config/files.py @@ -0,0 +1,18 @@ +""" +Simvue Config File Lists +======================== + +Contains lists of valid Simvue configuration file names. + +""" + +import pathlib + +CONFIG_FILE_NAMES: list[str] = ["simvue.toml", ".simvue.toml"] + +CONFIG_INI_FILE_NAMES: list[str] = [ + f'{pathlib.Path.cwd().joinpath("simvue.ini")}', + f'{pathlib.Path.home().joinpath(".simvue.ini")}', +] + +DEFAULT_OFFLINE_DIRECTORY: str = f"{pathlib.Path.home().joinpath('.simvue')}" diff --git a/simvue/config/parameters.py b/simvue/config/parameters.py index 2edad78e..a0c13b70 100644 --- a/simvue/config/parameters.py +++ b/simvue/config/parameters.py @@ -13,18 +13,13 @@ import typing import pathlib import http +import functools import simvue.models as sv_models from simvue.utilities import get_expiry from simvue.version import __version__ from simvue.api import get -CONFIG_FILE_NAMES: list[str] = ["simvue.toml", ".simvue.toml"] - -CONFIG_INI_FILE_NAMES: list[str] = [ - f'{pathlib.Path.cwd().joinpath("simvue.ini")}', - f'{pathlib.Path.home().joinpath(".simvue.ini")}', -] logger = logging.getLogger(__file__) @@ -47,18 +42,15 @@ def check_token(cls, v: typing.Any) -> str: raise AssertionError("Simvue token has expired") return value - @pydantic.model_validator(mode="after") @classmethod - def check_valid_server(cls, values: "ServerSpecifications") -> bool: - if os.environ.get("SIMVUE_NO_SERVER_CHECK"): - return values - + @functools.lru_cache + def _check_server(cls, token: str, url: str) -> None: headers: dict[str, str] = { - "Authorization": f"Bearer {values.token}", + "Authorization": f"Bearer {token}", "User-Agent": f"Simvue Python client {__version__}", } try: - response = get(f"{values.url}/api/version", headers) + response = get(f"{url}/api/version", headers) if response.status_code != http.HTTPStatus.OK or not response.json().get( "version" @@ -71,12 +63,25 @@ def check_valid_server(cls, values: "ServerSpecifications") -> bool: except Exception as err: raise AssertionError(f"Exception retrieving server version: {str(err)}") + @pydantic.model_validator(mode="after") + @classmethod + def check_valid_server(cls, values: "ServerSpecifications") -> bool: + if os.environ.get("SIMVUE_NO_SERVER_CHECK"): + return values + + cls._check_server(values.token, values.url) + return values class OfflineSpecifications(pydantic.BaseModel): cache: typing.Optional[pathlib.Path] = None + @pydantic.field_validator("cache") + @classmethod + def cache_to_str(cls, v: typing.Any) -> str: + return f"{v}" + class DefaultRunSpecifications(pydantic.BaseModel): name: typing.Optional[str] = None diff --git a/simvue/config/user.py b/simvue/config/user.py index e3225fc9..77b97aa6 100644 --- a/simvue/config/user.py +++ b/simvue/config/user.py @@ -20,14 +20,18 @@ import simvue.utilities as sv_util from simvue.config.parameters import ( - CONFIG_FILE_NAMES, - CONFIG_INI_FILE_NAMES, ClientGeneralOptions, DefaultRunSpecifications, ServerSpecifications, OfflineSpecifications, ) +from simvue.config.files import ( + CONFIG_FILE_NAMES, + CONFIG_INI_FILE_NAMES, + DEFAULT_OFFLINE_DIRECTORY, +) + logger = logging.getLogger(__name__) @@ -52,7 +56,7 @@ def _parse_ini_config(cls, ini_file: pathlib.Path) -> dict[str, dict[str, str]]: stacklevel=2, ) - config_dict: dict[str, dict[str, str]] = {"server": {}} + config_dict: dict[str, dict[str, str]] = {"server": {}, "offline": {}} with contextlib.suppress(Exception): parser = configparser.ConfigParser() @@ -61,6 +65,8 @@ def _parse_ini_config(cls, ini_file: pathlib.Path) -> dict[str, dict[str, str]]: config_dict["server"]["token"] = token if url := parser.get("server", "url"): config_dict["server"]["url"] = url + if offline_dir := parser.get("offline", "cache"): + config_dict["offline"]["cache"] = offline_dir return config_dict @@ -71,6 +77,24 @@ def fetch( server_url: typing.Optional[str] = None, server_token: typing.Optional[str] = None, ) -> "SimvueConfiguration": + """Retrieve the Simvue configuration from this project + + Will retrieve the configuration options set for this project either using + local or global configurations. + + Parameters + ---------- + server_url : str, optional + override the URL used for this session + server_token : str, optional + override the token used for this session + + Return + ------ + SimvueConfiguration + object containing configurations + + """ _config_dict: dict[str, dict[str, str]] = {} try: @@ -89,6 +113,16 @@ def fetch( _config_dict["server"] = _config_dict.get("server", {}) + _config_dict["offline"] = _config_dict.get("offline", {}) + + # Allow override of specification of offline directory via environment variable + if not (_default_dir := os.environ.get("SIMVUE_OFFLINE_DIRECTORY")): + _default_dir = _config_dict["offline"].get( + "cache", DEFAULT_OFFLINE_DIRECTORY + ) + + _config_dict["offline"]["cache"] = _default_dir + # Ranking of configurations for token and URl is: # Envionment Variables > Run Definition > Configuration File @@ -114,6 +148,7 @@ def fetch( @classmethod @functools.lru_cache def config_file(cls) -> pathlib.Path: + """Returns the path of top level configuration file used for the session""" _config_file: typing.Optional[pathlib.Path] = ( sv_util.find_first_instance_of_file( CONFIG_FILE_NAMES, check_user_space=True diff --git a/simvue/factory/proxy/offline.py b/simvue/factory/proxy/offline.py index 1e6583d9..cda666c7 100644 --- a/simvue/factory/proxy/offline.py +++ b/simvue/factory/proxy/offline.py @@ -9,9 +9,9 @@ import randomname from simvue.factory.proxy.base import SimvueBaseClass +from simvue.config.user import SimvueConfiguration from simvue.utilities import ( create_file, - get_offline_directory, prepare_for_api, skip_if_failed, ) @@ -32,7 +32,8 @@ def __init__( ) -> None: super().__init__(name, uniq_id, suppress_errors) - self._directory: str = os.path.join(get_offline_directory(), self._uuid) + _offline_dir = SimvueConfiguration.fetch().offline.cache + self._directory: str = os.path.join(_offline_dir, self._uuid) os.makedirs(self._directory, exist_ok=True) diff --git a/simvue/factory/proxy/remote.py b/simvue/factory/proxy/remote.py index f61ed0d3..3c8fdba4 100644 --- a/simvue/factory/proxy/remote.py +++ b/simvue/factory/proxy/remote.py @@ -3,7 +3,7 @@ import http if typing.TYPE_CHECKING: - from simvue.config import SimvueConfiguration + from simvue.config.user import SimvueConfiguration from simvue.api import get, post, put from simvue.factory.proxy.base import SimvueBaseClass diff --git a/simvue/run.py b/simvue/run.py index 12b1fbd7..5670c093 100644 --- a/simvue/run.py +++ b/simvue/run.py @@ -32,7 +32,7 @@ import psutil from pydantic import ValidationError -from .config import SimvueConfiguration +from .config.user import SimvueConfiguration import simvue.api as sv_api from .factory.dispatch import Dispatcher @@ -48,7 +48,6 @@ calculate_sha256, compare_alerts, skip_if_failed, - get_offline_directory, validate_timestamp, simvue_timestamp, ) @@ -405,7 +404,8 @@ def _offline_dispatch_callback( run_id: typing.Optional[str] = self._id, uuid: str = self._uuid, ) -> None: - if not os.path.exists((_offline_directory := get_offline_directory())): + _offline_directory = self.config.offline.cache + if not os.path.exists(_offline_directory): logger.error( f"Cannot write to offline directory '{_offline_directory}', directory not found." ) diff --git a/simvue/sender.py b/simvue/sender.py index 83290748..264a183c 100644 --- a/simvue/sender.py +++ b/simvue/sender.py @@ -12,7 +12,7 @@ from simvue.config.user import SimvueConfiguration from .factory.proxy.remote import Remote -from .utilities import create_file, get_offline_directory, remove_file +from .utilities import create_file, remove_file logger = logging.getLogger(__name__) @@ -88,7 +88,7 @@ def sender() -> str: """ Asynchronous upload of runs to Simvue server """ - directory = get_offline_directory() + directory = SimvueConfiguration.fetch().offline.cache # Clean up old runs after waiting 5 mins runs = glob.glob(f"{directory}/*/sent") diff --git a/simvue/utilities.py b/simvue/utilities.py index 6270a2d4..74a2eb8b 100644 --- a/simvue/utilities.py +++ b/simvue/utilities.py @@ -1,4 +1,3 @@ -import configparser import datetime import hashlib import logging @@ -16,6 +15,7 @@ from datetime import timezone + CHECKSUM_BLOCK_SIZE = 4096 EXTRAS: tuple[str, ...] = ("plot", "torch") @@ -256,30 +256,6 @@ def wrapper(self, *args, **kwargs) -> typing.Any: return wrapper -def get_offline_directory() -> str: - """ - Get directory for offline cache - """ - directory = None - - for filename in ( - os.path.join(os.path.expanduser("~"), ".simvue.ini"), - "simvue.ini", - ): - if not filename or not os.path.exists(filename): - continue - - with contextlib.suppress(Exception): - config = configparser.ConfigParser() - config.read(filename) - directory = config.get("offline", "cache") - - if not (directory := os.environ.get("SIMVUE_OFFLINE_DIRECTORY", directory)): - directory = os.path.join(os.path.expanduser("~"), ".simvue") - - return directory - - def create_file(filename: str) -> None: """ Create an empty file diff --git a/tests/refactor/conftest.py b/tests/refactor/conftest.py index 61214b10..f2f599b4 100644 --- a/tests/refactor/conftest.py +++ b/tests/refactor/conftest.py @@ -17,11 +17,11 @@ def __init__(self, level=logging.DEBUG): super().__init__(level) self.counts = [] self.captures = [] - + def emit(self, record): if len(self.captures) != len(self.counts): self.counts = [0] * len(self.captures) - + for i, capture in enumerate(self.captures): if capture in record.msg: self.counts[i] += 1 @@ -48,9 +48,9 @@ def create_test_run(request) -> typing.Generator[typing.Tuple[sv_run.Run, dict], @pytest.fixture -def create_test_run_offline(mocker: pytest_mock.MockerFixture, request) -> typing.Generator[typing.Tuple[sv_run.Run, dict], None, None]: +def create_test_run_offline(mocker: pytest_mock.MockerFixture, request, monkeypatch: pytest.MonkeyPatch) -> typing.Generator[typing.Tuple[sv_run.Run, dict], None, None]: with tempfile.TemporaryDirectory() as temp_d: - mocker.patch.object(simvue.utilities, "get_offline_directory", lambda *_: temp_d) + monkeypatch.setenv("SIMVUE_OFFLINE_DIRECTORY", temp_d) with sv_run.Run("offline") as run: yield run, setup_test_run(run, True, request) @@ -62,9 +62,9 @@ def create_plain_run(request) -> typing.Generator[typing.Tuple[sv_run.Run, dict] @pytest.fixture -def create_plain_run_offline(mocker: pytest_mock.MockerFixture, request) -> typing.Generator[typing.Tuple[sv_run.Run, dict], None, None]: +def create_plain_run_offline(mocker: pytest_mock.MockerFixture, request, monkeypatch: pytest.MonkeyPatch) -> typing.Generator[typing.Tuple[sv_run.Run, dict], None, None]: with tempfile.TemporaryDirectory() as temp_d: - mocker.patch.object(simvue.utilities, "get_offline_directory", lambda *_: temp_d) + monkeypatch.setenv("SIMVUE_OFFLINE_DIRECTORY", temp_d) with sv_run.Run("offline") as run: yield run, setup_test_run(run, False, request) diff --git a/tests/refactor/test_config.py b/tests/refactor/test_config.py index f7236a1d..4bc0cb4e 100644 --- a/tests/refactor/test_config.py +++ b/tests/refactor/test_config.py @@ -5,7 +5,7 @@ import pathlib import pytest_mock import tempfile -from simvue.config import SimvueConfiguration +from simvue.config.user import SimvueConfiguration @pytest.mark.config @@ -39,14 +39,13 @@ def test_config_setup( _tags: list[str] = ["tag-test", "other-tag"] # Deactivate the server checks for this test - monkeypatch.setenv("SIMVUE_NO_SERVER_CHECK", True) + monkeypatch.setenv("SIMVUE_NO_SERVER_CHECK", "True") + monkeypatch.delenv("SIMVUE_TOKEN", False) + monkeypatch.delenv("SIMVUE_URL", False) if use_env: monkeypatch.setenv("SIMVUE_TOKEN", _other_token) monkeypatch.setenv("SIMVUE_URL", _other_url) - else: - monkeypatch.delenv("SIMVUE_TOKEN", False) - monkeypatch.delenv("SIMVUE_URL", False) with tempfile.TemporaryDirectory() as temp_d: _config_file = None @@ -57,12 +56,18 @@ def test_config_setup( [server] url = "{_url}" token = "{_token}" + +[offline] +cache = "{temp_d}" """ else: _lines = f""" [server] url = {_url} token = {_token} + +[offline] +cache = {temp_d} """ if use_file == "extended": @@ -78,19 +83,19 @@ def test_config_setup( mocker.patch("simvue.config.parameters.get_expiry", lambda *_, **__: 1e10) mocker.patch("simvue.config.user.sv_util.find_first_instance_of_file", lambda *_, **__: _config_file) - import simvue.config + import simvue.config.user if not use_file and not use_env and not use_args: with pytest.raises(RuntimeError): - simvue.config.SimvueConfiguration.fetch() + simvue.config.user.SimvueConfiguration.fetch() return elif use_args: - _config = simvue.config.SimvueConfiguration.fetch( + _config = simvue.config.user.SimvueConfiguration.fetch( server_url=_arg_url, server_token=_arg_token ) else: - _config = simvue.config.SimvueConfiguration.fetch() + _config = simvue.config.user.SimvueConfiguration.fetch() if use_file: assert _config.config_file() == _config_file @@ -104,6 +109,7 @@ def test_config_setup( elif use_file: assert _config.server.url == _url assert _config.server.token == _token + assert _config.offline.cache == temp_d if use_file == "extended": assert _config.run.description == _description @@ -114,5 +120,5 @@ def test_config_setup( assert not _config.run.description assert not _config.run.tags - simvue.config.SimvueConfiguration.config_file.cache_clear() + simvue.config.user.SimvueConfiguration.config_file.cache_clear() diff --git a/tests/refactor/test_run_class.py b/tests/refactor/test_run_class.py index 1af7cc5e..b8ba4908 100644 --- a/tests/refactor/test_run_class.py +++ b/tests/refactor/test_run_class.py @@ -351,6 +351,7 @@ def test_bad_run_arguments() -> None: run.init("sdas", [34]) +@pytest.mark.run def test_set_folder_details(request: pytest.FixtureRequest) -> None: with sv_run.Run() as run: folder_name: str = "/simvue_unit_tests"