diff --git a/dandi/cli/base.py b/dandi/cli/base.py index 68981a231..79d96cb16 100644 --- a/dandi/cli/base.py +++ b/dandi/cli/base.py @@ -4,7 +4,6 @@ import click from .. import get_logger -from ..consts import known_instances lgr = get_logger() @@ -95,7 +94,6 @@ def dandiset_path_option(**kwargs): def instance_option(**kwargs): params = { "help": "DANDI instance to use", - "type": click.Choice(sorted(known_instances)), "default": "dandi", "show_default": True, "envvar": "DANDI_INSTANCE", diff --git a/dandi/cli/cmd_download.py b/dandi/cli/cmd_download.py index 7990a9449..3ac975e4d 100644 --- a/dandi/cli/cmd_download.py +++ b/dandi/cli/cmd_download.py @@ -3,8 +3,8 @@ import click from .base import ChoiceList, IntColonInt, instance_option, map_to_click_exceptions -from ..consts import known_instances, known_instances_rev from ..dandiarchive import _dandi_url_parser, parse_dandi_url +from ..utils import get_instance # The use of f-strings apparently makes this not a proper docstring, and so @@ -111,10 +111,11 @@ def download( from .. import download if dandi_instance is not None: + instance = get_instance(dandi_instance) if url: for u in url: parsed_url = parse_dandi_url(u) - if known_instances_rev.get(parsed_url.api_url) != dandi_instance: + if parsed_url.instance != instance: raise click.UsageError( f"{u} does not point to {dandi_instance!r} instance" ) @@ -127,15 +128,10 @@ def download( # No Dandiset here; leave `url` alone pass else: - instance = known_instances[dandi_instance] if instance.gui is not None: url = [f"{instance.gui}/#/dandiset/{dandiset_id}/draft"] - elif instance.api is not None: - url = [f"{instance.api}/dandisets/{dandiset_id}/"] else: - raise NotImplementedError( - f"Do not know how to construct URLs for {dandi_instance!r}" - ) + url = [f"{instance.api}/dandisets/{dandiset_id}/"] return download.download( url, diff --git a/dandi/cli/cmd_instances.py b/dandi/cli/cmd_instances.py index 29011007a..7c893187c 100644 --- a/dandi/cli/cmd_instances.py +++ b/dandi/cli/cmd_instances.py @@ -1,3 +1,4 @@ +from dataclasses import asdict import sys import click @@ -13,4 +14,9 @@ def instances(): """List known Dandi Archive instances that the CLI can interact with""" yaml = ruamel.yaml.YAML(typ="safe") yaml.default_flow_style = False - yaml.dump({k: v._asdict() for k, v in known_instances.items()}, sys.stdout) + instances = {} + for inst in known_instances.values(): + data = asdict(inst) + data.pop("name") + instances[inst.name] = data + yaml.dump(instances, sys.stdout) diff --git a/dandi/cli/tests/test_download.py b/dandi/cli/tests/test_download.py index f77af0841..0d44f41a3 100644 --- a/dandi/cli/tests/test_download.py +++ b/dandi/cli/tests/test_download.py @@ -101,7 +101,7 @@ def test_download_gui_instance_in_dandiset(mocker): r = runner.invoke(download, ["-i", "dandi"]) assert r.exit_code == 0 mock_download.assert_called_once_with( - ["https://gui.dandiarchive.org/#/dandiset/123456/draft"], + ["https://dandiarchive.org/#/dandiset/123456/draft"], os.curdir, existing="error", format="pyout", diff --git a/dandi/cli/tests/test_instances.py b/dandi/cli/tests/test_instances.py index fd96bec4a..dc3ce48a5 100644 --- a/dandi/cli/tests/test_instances.py +++ b/dandi/cli/tests/test_instances.py @@ -6,27 +6,17 @@ def test_cmd_instances(monkeypatch): - redirector_base = os.environ.get( - "DANDI_REDIRECTOR_BASE", "https://dandiarchive.org" - ) instancehost = os.environ.get("DANDI_INSTANCEHOST", "localhost") r = CliRunner().invoke(instances, []) assert r.exit_code == 0 assert r.output == ( "dandi:\n" " api: https://api.dandiarchive.org/api\n" - " gui: https://gui.dandiarchive.org\n" - f" redirector: {redirector_base}\n" + " gui: https://dandiarchive.org\n" "dandi-api-local-docker-tests:\n" f" api: http://{instancehost}:8000/api\n" f" gui: http://{instancehost}:8085\n" - " redirector: null\n" - "dandi-devel:\n" - " api: null\n" - " gui: https://gui-beta-dandiarchive-org.netlify.app\n" - " redirector: null\n" "dandi-staging:\n" " api: https://api-staging.dandiarchive.org/api\n" " gui: https://gui-staging.dandiarchive.org\n" - " redirector: null\n" ) diff --git a/dandi/consts.py b/dandi/consts.py index 7ac9be1e1..34391185a 100644 --- a/dandi/consts.py +++ b/dandi/consts.py @@ -1,6 +1,10 @@ +from __future__ import annotations + +from collections.abc import Iterator +from dataclasses import dataclass from enum import Enum import os -from typing import NamedTuple, Optional +from typing import Optional #: A list of metadata fields which dandi extracts from .nwb files. #: Additional fields (such as ``number_of_*``) might be added by @@ -92,40 +96,48 @@ class EmbargoStatus(Enum): dandiset_identifier_regex = f"^{DANDISET_ID_REGEX}$" -class DandiInstance(NamedTuple): +@dataclass(frozen=True) +class DandiInstance: + name: str gui: Optional[str] - redirector: Optional[str] - api: Optional[str] + api: str + + @property + def redirector(self) -> None: + # For "backwards compatibility" + return None + + def urls(self) -> Iterator[str]: + if self.gui is not None: + yield self.gui + yield self.api # So it could be easily mapped to external IP (e.g. from within VM) # to test against instance running outside of current environment instancehost = os.environ.get("DANDI_INSTANCEHOST", "localhost") -redirector_base = os.environ.get("DANDI_REDIRECTOR_BASE", "https://dandiarchive.org") - known_instances = { "dandi": DandiInstance( - "https://gui.dandiarchive.org", - redirector_base, + "dandi", + "https://dandiarchive.org", "https://api.dandiarchive.org/api", ), - "dandi-devel": DandiInstance( - "https://gui-beta-dandiarchive-org.netlify.app", - None, - None, - ), "dandi-staging": DandiInstance( + "dandi-staging", "https://gui-staging.dandiarchive.org", - None, "https://api-staging.dandiarchive.org/api", ), "dandi-api-local-docker-tests": DandiInstance( - f"http://{instancehost}:8085", None, f"http://{instancehost}:8000/api" + "dandi-api-local-docker-tests", + f"http://{instancehost}:8085", + f"http://{instancehost}:8000/api", ), } # to map back url: name -known_instances_rev = {vv: k for k, v in known_instances.items() for vv in v if vv} +known_instances_rev = { + vv: k for k, v in known_instances.items() for vv in v.urls() if vv +} file_operation_modes = [ "dry", diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index be8411604..673939658 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -45,8 +45,6 @@ ZARR_DELETE_BATCH_SIZE, DandiInstance, EmbargoStatus, - known_instances, - known_instances_rev, ) from .exceptions import HTTP404Error, NotFoundError, SchemaVersionError from .keyring import keyring_lookup, keyring_save @@ -56,6 +54,7 @@ check_dandi_version, chunked, ensure_datetime, + get_instance, is_interactive, is_page2_url, ) @@ -412,27 +411,36 @@ class DandiAPIClient(RESTFullAPIClient): """A client for interacting with a Dandi Archive server""" def __init__( - self, api_url: Optional[str] = None, token: Optional[str] = None + self, + api_url: Optional[str] = None, + token: Optional[str] = None, + dandi_instance: Optional[DandiInstance] = None, ) -> None: """ - Construct a client instance. + Construct a client instance for the given API URL or Dandi instance + (mutually exclusive options). If no URL or instance is supplied, the + instance specified by the :envvar:`DANDI_INSTANCE` environment variable + (default value: ``"dandi"``) is used. :param str api_url: Base API URL of the server to interact with. - - For DANDI production, use ``"https://api.dandiarchive.org/api"`` - - For DANDI staging, use ``"https://api-staging.dandiarchive.org/api"`` - - If no URL is supplied, the URL is looked up in `known_instances` using the value of the - :envvar:`DANDI_INSTANCE` environment variable (default value: ``"dandi"``). - :param str token: User API Key. Note that different instance APIs have different - keys. + - For DANDI production, use ``"https://api.dandiarchive.org/api"`` + - For DANDI staging, use + ``"https://api-staging.dandiarchive.org/api"`` + :param str token: User API Key. Note that different instance APIs have + different keys. """ - check_dandi_version() if api_url is None: - instance_name = os.environ.get("DANDI_INSTANCE", "dandi") - api_url = known_instances[instance_name].api - if api_url is None: - raise ValueError(f"No API URL for instance {instance_name!r}") + if dandi_instance is None: + instance_name = os.environ.get("DANDI_INSTANCE", "dandi") + dandi_instance = get_instance(instance_name) + api_url = dandi_instance.api + elif dandi_instance is not None: + raise ValueError("api_url and dandi_instance are mutually exclusive") + else: + dandi_instance = get_instance(api_url) super().__init__(api_url) + self.dandi_instance: DandiInstance = dandi_instance if token is not None: self.authenticate(token) @@ -450,9 +458,7 @@ def for_dandi_instance( If no token is supplied and ``authenticate`` is true, `dandi_authenticate()` is called on the instance before returning it. """ - if isinstance(instance, str): - instance = known_instances[instance] - client = cls(instance.api, token=token) + client = cls(dandi_instance=get_instance(instance), token=token) if token is None and authenticate: client.dandi_authenticate() return client @@ -527,19 +533,12 @@ def dandi_authenticate(self) -> None: break def _get_keyring_ids(self) -> tuple[str, str]: - try: - client_name = known_instances_rev[self.api_url] - except KeyError: - raise NotImplementedError("TODO client name derivation for keyring") + client_name = self.dandi_instance.name return (client_name, f"dandi-api-{client_name}") @property def _instance_id(self) -> str: - url = self.api_url.rstrip("/") - try: - return known_instances_rev[url].upper() - except KeyError: - return url + return self.dandi_instance.name.upper() def get_dandiset( self, dandiset_id: str, version_id: Optional[str] = None, lazy: bool = True diff --git a/dandi/dandiarchive.py b/dandi/dandiarchive.py index f09313b77..68cea6d05 100644 --- a/dandi/dandiarchive.py +++ b/dandi/dandiarchive.py @@ -35,7 +35,7 @@ from typing import Any, Dict, Iterable, Iterator, List, Optional, Tuple, cast from urllib.parse import unquote as urlunquote -from pydantic import AnyHttpUrl, BaseModel, parse_obj_as, validator +from pydantic import AnyHttpUrl, BaseModel, parse_obj_as import requests from . import get_logger @@ -44,6 +44,7 @@ PUBLISHED_VERSION_REGEX, RETRY_STATUSES, VERSION_REGEX, + DandiInstance, known_instances, ) from .dandiapi import BaseRemoteAsset, DandiAPIClient, RemoteDandiset @@ -59,13 +60,14 @@ class ParsedDandiURL(ABC, BaseModel): (Dandiset or asset(s)). Subclasses must implement `get_assets()`. Most methods take a ``client: DandiAPIClient`` argument, which must be a - `~dandi.dandiapi.DandiAPIClient` object for querying `api_url` (This is not - checked). Such a client instance can be obtained by calling + `~dandi.dandiapi.DandiAPIClient` object for querying `instance` (This is + not checked). Such a client instance can be obtained by calling `get_client()`, or an appropriate pre-existing client instance can be - passed instead.""" + passed instead. + """ - #: The base URL of the Dandi API service, without a trailing slash - api_url: AnyHttpUrl + #: The Dandi Archive instance that the URL points to + instance: DandiInstance #: The ID of the Dandiset given in the URL dandiset_id: Optional[str] #: The version of the Dandiset, if specified. If this is not set, the @@ -73,16 +75,18 @@ class ParsedDandiURL(ABC, BaseModel): #: `DandiAPIClient.get_dandiset()`. version_id: Optional[str] = None - @validator("api_url") - def _validate_api_url(cls, v: AnyHttpUrl) -> AnyHttpUrl: - return cast(AnyHttpUrl, parse_obj_as(AnyHttpUrl, v.rstrip("/"))) + @property + def api_url(self) -> AnyHttpUrl: + """The base URL of the Dandi API service, without a trailing slash""" + # Kept for backwards compatibility + return cast(AnyHttpUrl, parse_obj_as(AnyHttpUrl, self.instance.api.rstrip("/"))) def get_client(self) -> DandiAPIClient: """ Returns an unauthenticated `~dandi.dandiapi.DandiAPIClient` for - `api_url` + `instance` """ - return DandiAPIClient(self.api_url) + return DandiAPIClient(dandi_instance=self.instance) def get_dandiset( self, client: DandiAPIClient, lazy: bool = True @@ -550,9 +554,9 @@ class _dandi_url_parser: "DANDI:[/]", ), ( - re.compile(r"https?://dandiarchive\.org/.*"), + re.compile(r"https?://gui\.dandiarchive\.org/.*"), {"handle_redirect": "pass"}, - "https://dandiarchive.org/...", + "https://gui.dandiarchive.org/...", ), ( re.compile( @@ -652,12 +656,6 @@ class _dandi_url_parser: known_patterns = "Accepted resource identifier patterns:" + "\n - ".join( [""] + [display for _, _, display in known_urls] ) - map_to = {} - for (gui, redirector, api) in known_instances.values(): - if api: - for h in (gui, redirector): - if h: - map_to[h] = api @classmethod def parse( @@ -721,11 +719,7 @@ def parse( settings["map_instance"], ", ".join(known_instances) ) ) - known_instance = get_instance(settings["map_instance"]) - assert known_instance.api is not None - parsed_url.api_url = cast( - AnyHttpUrl, parse_obj_as(AnyHttpUrl, known_instance.api) - ) + parsed_url.instance = get_instance(settings["map_instance"]) continue # in this run we ignore and match further elif "instance_name" in groups: try: @@ -756,7 +750,7 @@ def parse( url_server, ): url_server = "https://gui-staging.dandiarchive.org" - server = cls.map_to.get(url_server, url_server) + instance = get_instance(url_server) # asset_type = groups.get("asset_type") dandiset_id = groups.get("dandiset_id") version_id = groups.get("version") @@ -773,52 +767,52 @@ def parse( if location: if glob: parsed_url = AssetGlobURL( - api_url=server, + instance=instance, dandiset_id=dandiset_id, version_id=version_id, path=location, ) elif location.endswith("/"): parsed_url = AssetFolderURL( - api_url=server, + instance=instance, dandiset_id=dandiset_id, version_id=version_id, path=location, ) else: parsed_url = AssetItemURL( - api_url=server, + instance=instance, dandiset_id=dandiset_id, version_id=version_id, path=location, ) elif asset_id: if dandiset_id is None: - parsed_url = BaseAssetIDURL(api_url=server, asset_id=asset_id) + parsed_url = BaseAssetIDURL(instance=instance, asset_id=asset_id) else: parsed_url = AssetIDURL( - api_url=server, + instance=instance, dandiset_id=dandiset_id, version_id=version_id, asset_id=asset_id, ) elif path: parsed_url = AssetPathPrefixURL( - api_url=server, + instance=instance, dandiset_id=dandiset_id, version_id=version_id, path=path, ) elif glob_param: parsed_url = AssetGlobURL( - api_url=server, + instance=instance, dandiset_id=dandiset_id, version_id=version_id, path=glob_param, ) else: parsed_url = DandisetURL( - api_url=server, + instance=instance, dandiset_id=dandiset_id, version_id=version_id, ) diff --git a/dandi/delete.py b/dandi/delete.py index 5b18096d6..4d2486c0d 100644 --- a/dandi/delete.py +++ b/dandi/delete.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from dataclasses import dataclass, field from operator import attrgetter from pathlib import Path @@ -5,7 +7,7 @@ import click -from .consts import DRAFT, ZARR_EXTENSIONS, dandiset_metadata_file +from .consts import DRAFT, ZARR_EXTENSIONS, DandiInstance, dandiset_metadata_file from .dandiapi import DandiAPIClient, RemoteAsset, RemoteDandiset from .dandiarchive import BaseAssetIDURL, DandisetURL, ParsedDandiURL, parse_dandi_url from .exceptions import NotFoundError @@ -28,16 +30,13 @@ class Deleter: def __bool__(self) -> bool: return self.deleting_dandiset or bool(self.remote_assets) - def set_dandiset(self, api_url: str, dandiset_id: str) -> bool: + def set_dandiset(self, instance: DandiInstance, dandiset_id: str) -> bool: """ Returns `False` if no action should be taken due to the Dandiset not existing """ if self.client is None: - # Strip the trailing slash so that dandi_authenticate can find the - # URL in known_instances_rev: - self.client = DandiAPIClient(api_url.rstrip("/")) - self.client.dandi_authenticate() + self.client = DandiAPIClient.for_dandi_instance(instance, authenticate=True) try: self.dandiset = self.client.get_dandiset(dandiset_id, DRAFT, lazy=False) except NotFoundError: @@ -45,7 +44,7 @@ def set_dandiset(self, api_url: str, dandiset_id: str) -> bool: return False else: raise - elif not is_same_url(self.client.api_url, api_url): + elif not is_same_url(self.client.api_url, instance.api): raise ValueError("Cannot delete assets from multiple API instances at once") else: assert self.dandiset is not None @@ -59,15 +58,19 @@ def add_asset(self, asset: RemoteAsset) -> None: if not any(a.identifier == asset.identifier for a in self.remote_assets): self.remote_assets.append(asset) - def register_dandiset(self, api_url: str, dandiset_id: str) -> None: - if not self.set_dandiset(api_url, dandiset_id): + def register_dandiset(self, instance: DandiInstance, dandiset_id: str) -> None: + if not self.set_dandiset(instance, dandiset_id): return self.deleting_dandiset = True def register_asset( - self, api_url: str, dandiset_id: str, version_id: str, asset_path: str + self, + instance: DandiInstance, + dandiset_id: str, + version_id: str, + asset_path: str, ) -> None: - if not self.set_dandiset(api_url, dandiset_id): + if not self.set_dandiset(instance, dandiset_id): return assert self.dandiset is not None try: @@ -82,9 +85,13 @@ def register_asset( self.add_asset(asset) def register_asset_folder( - self, api_url: str, dandiset_id: str, version_id: str, folder_path: str + self, + instance: DandiInstance, + dandiset_id: str, + version_id: str, + folder_path: str, ) -> None: - if not self.set_dandiset(api_url, dandiset_id): + if not self.set_dandiset(instance, dandiset_id): return any_assets = False assert self.dandiset is not None @@ -100,7 +107,7 @@ def register_assets_url(self, url: str, parsed_url: ParsedDandiURL) -> None: if isinstance(parsed_url, BaseAssetIDURL): raise ValueError("Cannot delete an asset identified by just an ID") assert parsed_url.dandiset_id is not None - if not self.set_dandiset(parsed_url.api_url, parsed_url.dandiset_id): + if not self.set_dandiset(parsed_url.instance, parsed_url.dandiset_id): return any_assets = False assert self.client is not None @@ -120,23 +127,23 @@ def register_url(self, url: str) -> None: " versions of a dandiset" ) assert parsed_url.dandiset_id is not None - self.register_dandiset(parsed_url.api_url, parsed_url.dandiset_id) + self.register_dandiset(parsed_url.instance, parsed_url.dandiset_id) else: if parsed_url.version_id is None: parsed_url.version_id = DRAFT self.register_assets_url(url, parsed_url) - def register_local_path_equivalent(self, instance_name: str, filepath: str) -> None: + def register_local_path_equivalent( + self, instance_name: str | DandiInstance, filepath: str + ) -> None: instance = get_instance(instance_name) - api_url = instance.api - assert api_url is not None dandiset_id, asset_path = find_local_asset(filepath) - if not self.set_dandiset(api_url, dandiset_id): + if not self.set_dandiset(instance, dandiset_id): return if asset_path.endswith("/"): - self.register_asset_folder(api_url, dandiset_id, DRAFT, asset_path) + self.register_asset_folder(instance, dandiset_id, DRAFT, asset_path) else: - self.register_asset(api_url, dandiset_id, DRAFT, asset_path) + self.register_asset(instance, dandiset_id, DRAFT, asset_path) def confirm(self) -> bool: if self.dandiset is None: @@ -182,7 +189,7 @@ def process_assets_debug(self) -> Iterator[Iterator[dict]]: def delete( paths: Iterable[str], - dandi_instance: str = "dandi", + dandi_instance: str | DandiInstance = "dandi", devel_debug: bool = False, jobs: Optional[int] = None, force: bool = False, diff --git a/dandi/move.py b/dandi/move.py index cee9cff10..42f66dd5b 100644 --- a/dandi/move.py +++ b/dandi/move.py @@ -12,6 +12,7 @@ from typing import NewType, Optional from . import get_logger +from .consts import DandiInstance from .dandiapi import DandiAPIClient, RemoteAsset, RemoteDandiset from .dandiarchive import DandisetURL, parse_dandi_url from .dandiset import Dandiset @@ -760,7 +761,7 @@ def move( dest: str, regex: bool = False, existing: str = "error", - dandi_instance: str = "dandi", + dandi_instance: str | DandiInstance = "dandi", dandiset: Path | str | None = None, work_on: str = "auto", devel_debug: bool = False, diff --git a/dandi/tests/fixtures.py b/dandi/tests/fixtures.py index 9334938df..4ffaec210 100644 --- a/dandi/tests/fixtures.py +++ b/dandi/tests/fixtures.py @@ -499,28 +499,26 @@ def docker_compose_setup() -> Iterator[Dict[str, str]]: class DandiAPI: api_key: str client: DandiAPIClient - instance: DandiInstance - instance_id: str + + @property + def instance(self) -> DandiInstance: + return self.client.dandi_instance + + @property + def instance_id(self) -> str: + return self.instance.name @property def api_url(self) -> str: - url = self.instance.api - assert isinstance(url, str) - return url + return self.instance.api @pytest.fixture(scope="session") def local_dandi_api(docker_compose_setup: Dict[str, str]) -> Iterator[DandiAPI]: - instance_id = "dandi-api-local-docker-tests" - instance = known_instances[instance_id] + instance = known_instances["dandi-api-local-docker-tests"] api_key = docker_compose_setup["django_api_key"] - with DandiAPIClient(api_url=instance.api, token=api_key) as client: - yield DandiAPI( - api_key=api_key, - client=client, - instance=instance, - instance_id=instance_id, - ) + with DandiAPIClient.for_dandi_instance(instance, token=api_key) as client: + yield DandiAPI(api_key=api_key, client=client) @dataclass diff --git a/dandi/tests/test_dandiapi.py b/dandi/tests/test_dandiapi.py index 2d7414453..ac8b04c06 100644 --- a/dandi/tests/test_dandiapi.py +++ b/dandi/tests/test_dandiapi.py @@ -266,10 +266,24 @@ def test_remote_asset_json_dict(text_dandiset: SampleDandiset) -> None: @responses.activate def test_check_schema_version_matches_default() -> None: + server_info = { + "schema_version": get_schema_version(), + "version": "0.0.0", + "services": { + "api": {"url": "https://test.nil/api"}, + }, + "cli-minimal-version": "0.0.0", + "cli-bad-versions": [], + } + responses.add( + responses.GET, + "https://test.nil/server-info", + json=server_info, + ) responses.add( responses.GET, "https://test.nil/api/info/", - json={"schema_version": get_schema_version()}, + json=server_info, ) client = DandiAPIClient("https://test.nil/api") client.check_schema_version() @@ -277,8 +291,24 @@ def test_check_schema_version_matches_default() -> None: @responses.activate def test_check_schema_version_mismatch() -> None: + server_info = { + "schema_version": "4.5.6", + "version": "0.0.0", + "services": { + "api": {"url": "https://test.nil/api"}, + }, + "cli-minimal-version": "0.0.0", + "cli-bad-versions": [], + } + responses.add( + responses.GET, + "https://test.nil/server-info", + json=server_info, + ) responses.add( - responses.GET, "https://test.nil/api/info/", json={"schema_version": "4.5.6"} + responses.GET, + "https://test.nil/api/info/", + json=server_info, ) client = DandiAPIClient("https://test.nil/api") with pytest.raises(SchemaVersionError) as excinfo: @@ -554,6 +584,19 @@ def test_get_asset_with_and_without_metadata( @responses.activate def test_retry_logging(caplog: pytest.LogCaptureFixture) -> None: + responses.add( + responses.GET, + "https://test.nil/server-info", + json={ + "schema_version": get_schema_version(), + "version": "0.0.0", + "services": { + "api": {"url": "https://test.nil/api"}, + }, + "cli-minimal-version": "0.0.0", + "cli-bad-versions": [], + }, + ) responses.add(responses.GET, "https://test.nil/api/info/", status=503) responses.add(responses.GET, "https://test.nil/api/info/", status=503) responses.add(responses.GET, "https://test.nil/api/info/", json={"foo": "bar"}) diff --git a/dandi/tests/test_dandiarchive.py b/dandi/tests/test_dandiarchive.py index 79a65d707..5bb72c69e 100644 --- a/dandi/tests/test_dandiarchive.py +++ b/dandi/tests/test_dandiarchive.py @@ -3,7 +3,7 @@ import pytest import responses -from dandi.consts import known_instances +from dandi.consts import DandiInstance, known_instances from dandi.dandiarchive import ( AssetFolderURL, AssetGlobURL, @@ -27,115 +27,125 @@ "url,parsed_url", [ # New DANDI web UI driven by DANDI API. - ( + pytest.param( "https://gui.dandiarchive.org/#/dandiset/000001", DandisetURL( - api_url=known_instances["dandi"].api, + instance=known_instances["dandi"], dandiset_id="000001", version_id=None, ), + marks=mark.skipif_no_network, ), - ( + pytest.param( "https://gui.dandiarchive.org/#/dandiset/000001/", DandisetURL( - api_url=known_instances["dandi"].api, + instance=known_instances["dandi"], dandiset_id="000001", version_id=None, ), + marks=mark.skipif_no_network, ), - ( + pytest.param( "https://gui.dandiarchive.org/#/dandiset/000001/0.201104.2302", DandisetURL( - api_url=known_instances["dandi"].api, + instance=known_instances["dandi"], dandiset_id="000001", version_id="0.201104.2302", ), + marks=mark.skipif_no_network, ), - ( + pytest.param( "https://gui.dandiarchive.org/#/dandiset/000001/0.201104.2302/", DandisetURL( - api_url=known_instances["dandi"].api, + instance=known_instances["dandi"], dandiset_id="000001", version_id="0.201104.2302", ), + marks=mark.skipif_no_network, ), - ( + pytest.param( "https://gui.dandiarchive.org/#/dandiset/000001/0.201104.2302/files", DandisetURL( - api_url=known_instances["dandi"].api, + instance=known_instances["dandi"], dandiset_id="000001", version_id="0.201104.2302", ), + marks=mark.skipif_no_network, ), - ( + pytest.param( "https://gui.dandiarchive.org/#/dandiset/000001/draft", DandisetURL( - api_url=known_instances["dandi"].api, + instance=known_instances["dandi"], dandiset_id="000001", version_id="draft", ), + marks=mark.skipif_no_network, ), - ( + pytest.param( "https://gui.dandiarchive.org/dandiset/000001", DandisetURL( - api_url=known_instances["dandi"].api, + instance=known_instances["dandi"], dandiset_id="000001", version_id=None, ), + marks=mark.skipif_no_network, ), - ( + pytest.param( "https://gui.dandiarchive.org/dandiset/000001/0.201104.2302", DandisetURL( - api_url=known_instances["dandi"].api, + instance=known_instances["dandi"], dandiset_id="000001", version_id="0.201104.2302", ), + marks=mark.skipif_no_network, ), - ( + pytest.param( "https://gui.dandiarchive.org/dandiset/000001/0.201104.2302/files", DandisetURL( - api_url=known_instances["dandi"].api, + instance=known_instances["dandi"], dandiset_id="000001", version_id="0.201104.2302", ), + marks=mark.skipif_no_network, ), - ( + pytest.param( "https://gui.dandiarchive.org/dandiset/000001/draft", DandisetURL( - api_url=known_instances["dandi"].api, + instance=known_instances["dandi"], dandiset_id="000001", version_id="draft", ), + marks=mark.skipif_no_network, ), - pytest.param( + ( "DANDI:000027", DandisetURL( - api_url=known_instances["dandi"].api, + instance=known_instances["dandi"], dandiset_id="000027", version_id=None, ), ), - pytest.param( + ( "DANDI:000027/0.210831.2033", DandisetURL( - api_url=known_instances["dandi"].api, + instance=known_instances["dandi"], dandiset_id="000027", version_id="0.210831.2033", ), ), - pytest.param( + ( "DANDI:000027/draft", DandisetURL( - api_url=known_instances["dandi"].api, + instance=known_instances["dandi"], dandiset_id="000027", version_id="draft", ), ), # lower cased - pytest.param( + ( "dandi:000027/0.210831.2033", DandisetURL( - api_url=known_instances["dandi"].api, + instance=known_instances["dandi"], dandiset_id="000027", version_id="0.210831.2033", ), @@ -143,7 +153,7 @@ ( "http://localhost:8000/api/dandisets/000002/", DandisetURL( - api_url="http://localhost:8000/api", + instance=known_instances["dandi-api-local-docker-tests"], dandiset_id="000002", version_id=None, ), @@ -151,7 +161,7 @@ ( "http://localhost:8000/api/dandisets/000002", DandisetURL( - api_url="http://localhost:8000/api", + instance=known_instances["dandi-api-local-docker-tests"], dandiset_id="000002", version_id=None, ), @@ -159,7 +169,7 @@ ( "http://localhost:8000/api/dandisets/000002/versions/draft", DandisetURL( - api_url="http://localhost:8000/api", + instance=known_instances["dandi-api-local-docker-tests"], dandiset_id="000002", version_id="draft", ), @@ -167,7 +177,7 @@ ( "http://localhost:8000/api/dandisets/000002/versions/draft/", DandisetURL( - api_url="http://localhost:8000/api", + instance=known_instances["dandi-api-local-docker-tests"], dandiset_id="000002", version_id="draft", ), @@ -175,46 +185,49 @@ ( "http://localhost:8085/dandiset/000002", DandisetURL( - api_url="http://localhost:8000/api", + instance=known_instances["dandi-api-local-docker-tests"], dandiset_id="000002", version_id=None, ), ), - ( + pytest.param( "https://gui.dandiarchive.org/#/dandiset/000001/files" "?location=%2Fsub-anm369962", AssetItemURL( - api_url=known_instances["dandi"].api, + instance=known_instances["dandi"], dandiset_id="000001", version_id=None, path="sub-anm369962", ), + marks=mark.skipif_no_network, ), - ( + pytest.param( "https://gui.dandiarchive.org/#/dandiset/000006/0.200714.1807/files" "?location=%2Fsub-anm369962", AssetItemURL( - api_url=known_instances["dandi"].api, + instance=known_instances["dandi"], dandiset_id="000006", version_id="0.200714.1807", path="sub-anm369962", ), + marks=mark.skipif_no_network, ), - ( + pytest.param( "https://gui.dandiarchive.org/#/dandiset/001001/draft/files" "?location=sub-RAT123%2F", AssetFolderURL( - api_url=known_instances["dandi"].api, + instance=known_instances["dandi"], dandiset_id="001001", version_id="draft", path="sub-RAT123/", ), + marks=mark.skipif_no_network, ), # by direct instance name ad-hoc URI instance:ID[@version][/path] ( "dandi://dandi-api-local-docker-tests/000002@draft", DandisetURL( - api_url=known_instances["dandi-api-local-docker-tests"].api, + instance=known_instances["dandi-api-local-docker-tests"], dandiset_id="000002", version_id="draft", ), @@ -222,7 +235,7 @@ ( "dandi://dandi-api-local-docker-tests/000002@draft/path", AssetItemURL( - api_url=known_instances["dandi-api-local-docker-tests"].api, + instance=known_instances["dandi-api-local-docker-tests"], dandiset_id="000002", version_id="draft", path="path", @@ -231,7 +244,7 @@ ( "dandi://dandi-api-local-docker-tests/000002/path", AssetItemURL( - api_url=known_instances["dandi-api-local-docker-tests"].api, + instance=known_instances["dandi-api-local-docker-tests"], dandiset_id="000002", version_id=None, path="path", @@ -240,7 +253,7 @@ ( # test on "public" instance and have trailing / to signal the folder "dandi://dandi/000002/path/", AssetFolderURL( - api_url=known_instances["dandi"].api, + instance=known_instances["dandi"], dandiset_id="000002", version_id=None, path="path/", @@ -250,7 +263,7 @@ "https://api.dandiarchive.org/api/dandisets/000003/versions/draft" "/assets/0a748f90-d497-4a9c-822e-9c63811db412/download/", AssetIDURL( - api_url="https://api.dandiarchive.org/api", + instance=known_instances["dandi"], dandiset_id="000003", version_id="draft", asset_id="0a748f90-d497-4a9c-822e-9c63811db412", @@ -260,7 +273,7 @@ "https://api.dandiarchive.org/api/dandisets/000003/versions/draft" "/assets/0a748f90-d497-4a9c-822e-9c63811db412/download", AssetIDURL( - api_url="https://api.dandiarchive.org/api", + instance=known_instances["dandi"], dandiset_id="000003", version_id="draft", asset_id="0a748f90-d497-4a9c-822e-9c63811db412", @@ -270,7 +283,7 @@ "https://api.dandiarchive.org/api" "/assets/0a748f90-d497-4a9c-822e-9c63811db412/download/", BaseAssetIDURL( - api_url="https://api.dandiarchive.org/api", + instance=known_instances["dandi"], asset_id="0a748f90-d497-4a9c-822e-9c63811db412", ), ), @@ -278,7 +291,7 @@ "https://api.dandiarchive.org/api" "/assets/0a748f90-d497-4a9c-822e-9c63811db412/download", BaseAssetIDURL( - api_url="https://api.dandiarchive.org/api", + instance=known_instances["dandi"], asset_id="0a748f90-d497-4a9c-822e-9c63811db412", ), ), @@ -286,7 +299,7 @@ "https://api.dandiarchive.org/api/dandisets/000003/versions/draft" "/assets/?path=sub-YutaMouse20", AssetPathPrefixURL( - api_url="https://api.dandiarchive.org/api", + instance=known_instances["dandi"], dandiset_id="000003", version_id="draft", path="sub-YutaMouse20", @@ -295,7 +308,7 @@ ( "https://gui-staging.dandiarchive.org/#/dandiset/000018", DandisetURL( - api_url=known_instances["dandi-staging"].api, + instance=known_instances["dandi-staging"], dandiset_id="000018", version_id=None, ), @@ -304,25 +317,26 @@ "https://deploy-preview-854--gui-dandiarchive-org.netlify.app" "/#/dandiset/000018", DandisetURL( - api_url=known_instances["dandi-staging"].api, + instance=known_instances["dandi-staging"], dandiset_id="000018", version_id=None, ), ), - ( + pytest.param( "https://gui.dandiarchive.org/#/dandiset/001001/draft/files" "?location=sub-RAT123/*.nwb", AssetItemURL( - api_url=known_instances["dandi"].api, + instance=known_instances["dandi"], dandiset_id="001001", version_id="draft", path="sub-RAT123/*.nwb", ), + marks=mark.skipif_no_network, ), ( "dandi://dandi-api-local-docker-tests/000002/f*/bar.nwb", AssetItemURL( - api_url=known_instances["dandi-api-local-docker-tests"].api, + instance=known_instances["dandi-api-local-docker-tests"], dandiset_id="000002", version_id=None, path="f*/bar.nwb", @@ -332,7 +346,7 @@ "https://api.dandiarchive.org/api/dandisets/000003/versions/draft" "/assets/?glob=sub-YutaMouse*", AssetGlobURL( - api_url="https://api.dandiarchive.org/api", + instance=known_instances["dandi"], dandiset_id="000003", version_id="draft", path="sub-YutaMouse*", @@ -347,20 +361,21 @@ def test_parse_api_url(url: str, parsed_url: ParsedDandiURL) -> None: @pytest.mark.parametrize( "url,parsed_url", [ - ( + pytest.param( "https://gui.dandiarchive.org/#/dandiset/001001/draft/files" "?location=sub-RAT123/*.nwb", AssetGlobURL( - api_url=known_instances["dandi"].api, + instance=known_instances["dandi"], dandiset_id="001001", version_id="draft", path="sub-RAT123/*.nwb", ), + marks=mark.skipif_no_network, ), ( "dandi://dandi-api-local-docker-tests/000002/f*/bar.nwb", AssetGlobURL( - api_url=known_instances["dandi-api-local-docker-tests"].api, + instance=known_instances["dandi-api-local-docker-tests"], dandiset_id="000002", version_id=None, path="f*/bar.nwb", @@ -371,7 +386,7 @@ def test_parse_api_url(url: str, parsed_url: ParsedDandiURL) -> None: "https://api.dandiarchive.org/api/dandisets/000003/versions/draft" "/assets/?path=sub-YutaMouse*", AssetPathPrefixURL( - api_url="https://api.dandiarchive.org/api", + instance=known_instances["dandi"], dandiset_id="000003", version_id="draft", path="sub-YutaMouse*", @@ -381,7 +396,7 @@ def test_parse_api_url(url: str, parsed_url: ParsedDandiURL) -> None: "https://api.dandiarchive.org/api/dandisets/000003/versions/draft" "/assets/?glob=sub-YutaMouse*", AssetGlobURL( - api_url="https://api.dandiarchive.org/api", + instance=known_instances["dandi"], dandiset_id="000003", version_id="draft", path="sub-YutaMouse*", @@ -416,7 +431,7 @@ def test_parse_dandi_url_unknown_instance() -> None: parse_dandi_url("dandi://not-an-instance/000001") assert str(excinfo.value) == ( "Unknown instance 'not-an-instance'. Valid instances: dandi," - " dandi-api-local-docker-tests, dandi-devel, dandi-staging" + " dandi-api-local-docker-tests, dandi-staging" ) @@ -436,16 +451,22 @@ def test_follow_redirect() -> None: @responses.activate def test_parse_gui_new_redirect() -> None: - redirector_base = known_instances["dandi"].redirector + responses.add( + responses.HEAD, + "https://gui.dandiarchive.org/#/dandiset/000003", + status=302, + headers={"Location": "https://dandiarchive.org/"}, + ) + responses.add(responses.HEAD, "https://dandiarchive.org/") responses.add( responses.GET, - f"{redirector_base}/server-info", + "https://dandiarchive.org/server-info", json={ "version": "1.2.0", "cli-minimal-version": "0.6.0", "cli-bad-versions": [], "services": { - "webui": {"url": "https://gui.dandirchive.org"}, + "webui": {"url": "https://dandiarchive.org"}, "api": {"url": "https://api.dandiarchive.org/api"}, "jupyterhub": {"url": "https://hub.dandiarchive.org"}, }, @@ -454,12 +475,70 @@ def test_parse_gui_new_redirect() -> None: assert parse_dandi_url( "https://gui.dandiarchive.org/#/dandiset/000003" ) == DandisetURL( - api_url="https://api.dandiarchive.org/api", + instance=known_instances["dandi"], dandiset_id="000003", version_id=None, ) +@responses.activate +def test_parse_arbitrary_gui_url() -> None: + responses.add( + responses.GET, + "https://example.test/server-info", + json={ + "version": "1.2.0", + "cli-minimal-version": "0.6.0", + "cli-bad-versions": [], + "services": { + "webui": {"url": "https://example.test"}, + "api": {"url": "https://api.example.test/api"}, + }, + }, + ) + assert parse_dandi_url("https://example.test/dandiset/000123") == DandisetURL( + instance=DandiInstance( + name="api.example.test", + gui="https://example.test", + api="https://api.example.test/api", + ), + dandiset_id="000123", + version_id=None, + ) + + +@responses.activate +def test_parse_arbitrary_api_url() -> None: + responses.add( + responses.GET, + "https://api.example.test/server-info", + status=404, + ) + responses.add( + responses.GET, + "https://api.example.test/api/info/", + json={ + "version": "1.2.0", + "cli-minimal-version": "0.6.0", + "cli-bad-versions": [], + "services": { + "api": {"url": "https://api.example.test/api"}, + }, + }, + ) + assert parse_dandi_url( + "https://api.example.test/api/dandiset/000123" + ) == DandisetURL( + instance=DandiInstance( + name="api.example.test", + gui=None, + api="https://api.example.test/api", + ), + dandiset_id="000123", + version_id=None, + ) + + @pytest.mark.parametrize("version_suffix", ["", "@draft", "@0.999999.9999"]) def test_get_nonexistent_dandiset( local_dandi_api: DandiAPI, version_suffix: str diff --git a/dandi/tests/test_download.py b/dandi/tests/test_download.py index 5c9ed2433..46f2e491a 100644 --- a/dandi/tests/test_download.py +++ b/dandi/tests/test_download.py @@ -298,7 +298,7 @@ def test_download_metadata404(text_dandiset: SampleDandiset, tmp_path: Path) -> statuses = list( Downloader( url=DandisetURL( - api_url=text_dandiset.client.api_url, + instance=text_dandiset.api.instance, dandiset_id=text_dandiset.dandiset.identifier, version_id=text_dandiset.dandiset.version_id, ), diff --git a/dandi/tests/test_utils.py b/dandi/tests/test_utils.py index 42deff25e..b55cd9630 100644 --- a/dandi/tests/test_utils.py +++ b/dandi/tests/test_utils.py @@ -1,7 +1,6 @@ from __future__ import annotations import inspect -import os import os.path as op from pathlib import Path import time @@ -17,6 +16,7 @@ from ..consts import DandiInstance, known_instances from ..exceptions import BadCliVersionError, CliVersionTooOldError from ..utils import ( + _get_instance, ensure_datetime, ensure_strtime, find_files, @@ -169,14 +169,11 @@ def test_flatten() -> None: ] -redirector_base = known_instances["dandi"].redirector - - @responses.activate def test_get_instance_dandi_with_api() -> None: responses.add( responses.GET, - f"{redirector_base}/server-info", + "https://api.dandiarchive.org/api/info/", json={ "version": "1.0.0", "cli-minimal-version": "0.5.0", @@ -188,9 +185,10 @@ def test_get_instance_dandi_with_api() -> None: }, }, ) + _get_instance.cache_clear() assert get_instance("dandi") == DandiInstance( + name="dandi", gui="https://gui.dandi", - redirector=redirector_base, api="https://api.dandi", ) @@ -211,9 +209,10 @@ def test_get_instance_url() -> None: }, }, ) + _get_instance.cache_clear() assert get_instance("https://example.dandi/") == DandiInstance( + name="api.dandi", gui="https://gui.dandi", - redirector="https://example.dandi/", api="https://api.dandi", ) @@ -234,6 +233,7 @@ def test_get_instance_cli_version_too_old() -> None: }, }, ) + _get_instance.cache_clear() with pytest.raises(CliVersionTooOldError) as excinfo: get_instance("https://example.dandi/") assert str(excinfo.value) == ( @@ -258,6 +258,7 @@ def test_get_instance_bad_cli_version() -> None: }, }, ) + _get_instance.cache_clear() with pytest.raises(BadCliVersionError) as excinfo: get_instance("https://example.dandi/") assert str(excinfo.value) == ( @@ -270,23 +271,29 @@ def test_get_instance_bad_cli_version() -> None: def test_get_instance_id_bad_response() -> None: responses.add( responses.GET, - f"{redirector_base}/server-info", + "https://dandiarchive.org/server-info", body="404 -- not found", status=404, ) + _get_instance.cache_clear() assert get_instance("dandi") is known_instances["dandi"] @responses.activate def test_get_instance_known_url_bad_response() -> None: - assert redirector_base is not None responses.add( responses.GET, - f"{redirector_base}/server-info", + "https://dandiarchive.org/server-info", body="404 -- not found", status=404, ) - assert get_instance(redirector_base) is known_instances["dandi"] + _get_instance.cache_clear() + assert get_instance("https://dandiarchive.org") is known_instances["dandi"] + + +def test_get_known_instance_by_api() -> None: + _get_instance.cache_clear() + assert get_instance("https://api.dandiarchive.org/api/") == known_instances["dandi"] @responses.activate @@ -297,6 +304,7 @@ def test_get_instance_unknown_url_bad_response() -> None: body="404 -- not found", status=404, ) + _get_instance.cache_clear() with pytest.raises(RuntimeError) as excinfo: get_instance("https://dandi.nil") assert str(excinfo.value) == ( @@ -321,31 +329,75 @@ def test_get_instance_bad_version_from_server() -> None: }, }, ) + _get_instance.cache_clear() with pytest.raises(ValueError) as excinfo: get_instance("https://example.dandi/") assert str(excinfo.value).startswith( - "https://example.dandi/ returned an incorrectly formatted version;" + "https://example.dandi returned an incorrectly formatted version;" " please contact that server's administrators: " ) assert "foobar" in str(excinfo.value) def test_get_instance_actual_dandi() -> None: - inst = get_instance("dandi") - assert inst.api is not None + _get_instance.cache_clear() + get_instance("dandi") + + +@responses.activate +def test_get_instance_arbitrary_gui_url() -> None: + responses.add( + responses.GET, + "https://example.test/server-info", + json={ + "version": "1.2.0", + "cli-minimal-version": "0.6.0", + "cli-bad-versions": [], + "services": { + "webui": {"url": "https://example.test"}, + "api": {"url": "https://api.example.test/api"}, + }, + }, + ) + _get_instance.cache_clear() + assert get_instance("https://example.test/") == DandiInstance( + name="api.example.test", + gui="https://example.test", + api="https://api.example.test/api", + ) -if "DANDI_REDIRECTOR_BASE" in os.environ: - using_docker = pytest.mark.usefixtures("local_dandi_api") -else: - using_docker = mark.skipif_no_network +@responses.activate +def test_get_instance_arbitrary_api_url() -> None: + responses.add( + responses.GET, + "https://api.example.test/server-info", + status=404, + ) + responses.add( + responses.GET, + "https://api.example.test/api/info/", + json={ + "version": "1.2.0", + "cli-minimal-version": "0.6.0", + "cli-bad-versions": [], + "services": { + "api": {"url": "https://api.example.test/api"}, + }, + }, + ) + _get_instance.cache_clear() + assert get_instance("https://api.example.test/api/") == DandiInstance( + name="api.example.test", + gui=None, + api="https://api.example.test/api", + ) @pytest.mark.xfail(reason="https://github.com/dandi/dandi-archive/issues/1045") -@pytest.mark.redirector -@using_docker +@mark.skipif_no_network def test_server_info() -> None: - r = requests.get(f"{redirector_base}/server-info") + r = requests.get("https://dandiarchive.org/server-info") r.raise_for_status() data = r.json() assert "version" in data diff --git a/dandi/upload.py b/dandi/upload.py index 1513c4f6f..ac97ee463 100644 --- a/dandi/upload.py +++ b/dandi/upload.py @@ -14,7 +14,12 @@ from packaging.version import Version from . import __version__, lgr -from .consts import DRAFT, dandiset_identifier_regex, dandiset_metadata_file +from .consts import ( + DRAFT, + DandiInstance, + dandiset_identifier_regex, + dandiset_metadata_file, +) from .dandiapi import RemoteAsset from .exceptions import NotFoundError, UploadError from .files import ( @@ -25,7 +30,7 @@ ZarrAsset, ) from .misctypes import Digest -from .utils import ensure_datetime, get_instance, pluralize +from .utils import ensure_datetime, pluralize from .validate_types import Severity if TYPE_CHECKING: @@ -40,7 +45,7 @@ def upload( paths: Optional[List[Union[str, Path]]] = None, existing: str = "refresh", validation: str = "require", - dandi_instance: str = "dandi", + dandi_instance: str | DandiInstance = "dandi", allow_any_path: bool = False, upload_dandiset_metadata: bool = False, devel_debug: bool = False, @@ -62,15 +67,11 @@ def upload( " paths. Use 'dandi download' or 'organize' first." ) - instance = get_instance(dandi_instance) - assert instance.api is not None - api_url = instance.api - with ExitStack() as stack: # We need to use the client as a context manager in order to ensure the # session gets properly closed. Otherwise, pytest sometimes complains # under obscure conditions. - client = stack.enter_context(DandiAPIClient(api_url)) + client = stack.enter_context(DandiAPIClient.for_dandi_instance(dandi_instance)) client.check_schema_version() client.dandi_authenticate() diff --git a/dandi/utils.py b/dandi/utils.py index b522be3fd..2e2ee9025 100644 --- a/dandi/utils.py +++ b/dandi/utils.py @@ -2,6 +2,7 @@ from bisect import bisect import datetime +from functools import lru_cache from importlib.metadata import version as importlib_version import inspect import io @@ -29,9 +30,10 @@ TypeVar, Union, ) -from urllib.parse import parse_qs, urlparse +from urllib.parse import parse_qs, urlparse, urlunparse import dateutil.parser +from pydantic import AnyHttpUrl, BaseModel, Field import requests import ruamel.yaml from semantic_version import Version @@ -573,40 +575,79 @@ def delayed(*args, **kwargs): return joblib.delayed(*args, **kwargs) -def get_instance(dandi_instance_id: str) -> DandiInstance: - if dandi_instance_id.lower().startswith(("http://", "https://")): +class ServiceURL(BaseModel): + url: AnyHttpUrl + + +class ServerServices(BaseModel): + api: ServiceURL + webui: Optional[ServiceURL] = None + jupyterhub: Optional[ServiceURL] = None + + +class ServerInfo(BaseModel): + # schema_version: str + # schema_url: str + version: str + services: ServerServices + cli_minimal_version: str = Field(alias="cli-minimal-version") + cli_bad_versions: List[str] = Field(alias="cli-bad-versions") + + +def get_instance(dandi_instance_id: str | DandiInstance) -> DandiInstance: + dandi_id = None + redirector_url = None + if isinstance(dandi_instance_id, DandiInstance): + instance = dandi_instance_id + dandi_id = instance.name + elif dandi_instance_id.lower().startswith(("http://", "https://")): redirector_url = dandi_instance_id - dandi_id = known_instances_rev.get(redirector_url) + dandi_id = known_instances_rev.get(redirector_url.rstrip("/")) if dandi_id is not None: instance = known_instances[dandi_id] else: instance = None + bits = urlparse(redirector_url) + redirector_url = urlunparse((bits[0], bits[1], "", "", "", "")) else: - instance = known_instances[dandi_instance_id] - if instance.redirector is None: - return instance - else: - redirector_url = instance.redirector + dandi_id = dandi_instance_id + instance = known_instances[dandi_id] + if redirector_url is None: + assert instance is not None + return _get_instance(instance.api.rstrip("/"), True, instance, dandi_id) + else: + return _get_instance(redirector_url.rstrip("/"), False, instance, dandi_id) + + +@lru_cache +def _get_instance( + url: str, is_api: bool, instance: Optional[DandiInstance], dandi_id: Optional[str] +) -> DandiInstance: try: - r = requests.get(redirector_url.rstrip("/") + "/server-info") + if is_api: + r = requests.get(f"{url}/info/") + else: + r = requests.get(f"{url}/server-info") + if r.status_code == 404: + r = requests.get(f"{url}/api/info/") r.raise_for_status() + server_info = ServerInfo.parse_obj(r.json()) except Exception as e: - lgr.warning("Request to %s failed (%s)", redirector_url, str(e)) + lgr.warning("Request to %s failed (%s)", url, str(e)) if instance is not None: lgr.warning("Using hard-coded URLs") return instance else: raise RuntimeError( - f"Could not retrieve server info from {redirector_url}," + f"Could not retrieve server info from {url}," " and client does not recognize URL" ) - server_info = r.json() try: - minversion = Version(server_info["cli-minimal-version"]) - bad_versions = [Version(v) for v in server_info["cli-bad-versions"]] + minversion = Version(server_info.cli_minimal_version) + bad_versions = [Version(v) for v in server_info.cli_bad_versions] except ValueError as e: raise ValueError( - f"{redirector_url} returned an incorrectly formatted version;" + f"{url} returned an incorrectly formatted version;" f" please contact that server's administrators: {e}" ) our_version = Version(__version__) @@ -614,29 +655,21 @@ def get_instance(dandi_instance_id: str) -> DandiInstance: raise CliVersionTooOldError(our_version, minversion, bad_versions) if our_version in bad_versions: raise BadCliVersionError(our_version, minversion, bad_versions) - # note: service: url, not a full record - services = { - name: (rec or {}).get( - "url" - ) # note: somehow was ending up with {"girder": None} - for name, rec in server_info.get("services", {}).items() - } - for k, v in list(services.items()): - if v is not None: - services[k] = v.rstrip("/") - if services.get("api"): - return DandiInstance( - gui=services.get("webui"), - redirector=redirector_url, - api=services.get("api"), - ) - else: - raise RuntimeError( - "redirector's server-info returned unknown set of services keys: " - + ", ".join( - k for k, v in server_info.get("services", {}).items() if v is not None - ) - ) + api_url = server_info.services.api.url + if dandi_id is None: + dandi_id = api_url.host + assert dandi_id is not None + if api_url.port is not None: + if ":" in dandi_id: + dandi_id = f"[{dandi_id}]" + dandi_id += f":{api_url.port}" + return DandiInstance( + name=dandi_id, + gui=str(server_info.services.webui.url) + if server_info.services.webui is not None + else None, + api=str(api_url), + ) def is_url(s: str) -> bool: diff --git a/docs/source/cmdline/delete.rst b/docs/source/cmdline/delete.rst index 0dc4d73c7..97dca2d97 100644 --- a/docs/source/cmdline/delete.rst +++ b/docs/source/cmdline/delete.rst @@ -19,9 +19,10 @@ Options Force deletion without requesting interactive confirmation -.. option:: -i, --dandi-instance +.. option:: -i, --dandi-instance - DANDI instance to delete assets & Dandisets from [default: ``dandi``] + DANDI instance (either a base URL or a known instance name) to delete + assets & Dandisets from [default: ``dandi``] .. option:: --skip-missing diff --git a/docs/source/cmdline/download.rst b/docs/source/cmdline/download.rst index 2ce500904..e068194a4 100644 --- a/docs/source/cmdline/download.rst +++ b/docs/source/cmdline/download.rst @@ -27,9 +27,10 @@ Options Choose the format/frontend for output [default: ``pyout``] -.. option:: -i, --dandi-instance +.. option:: -i, --dandi-instance - DANDI instance to download from [default: ``dandi``] + DANDI instance (either a base URL or a known instance name) to download + from [default: ``dandi``] .. option:: -J, --jobs N[:M] diff --git a/docs/source/cmdline/instances.rst b/docs/source/cmdline/instances.rst index 5b325109e..3bdbb1f79 100644 --- a/docs/source/cmdline/instances.rst +++ b/docs/source/cmdline/instances.rst @@ -16,16 +16,9 @@ Example output: dandi: api: https://api.dandiarchive.org/api gui: https://gui.dandiarchive.org - redirector: https://dandiarchive.org dandi-api-local-docker-tests: api: http://localhost:8000/api gui: http://localhost:8085 - redirector: null - dandi-devel: - api: null - gui: https://gui-beta-dandiarchive-org.netlify.app - redirector: null dandi-staging: api: https://api-staging.dandiarchive.org/api gui: https://gui-staging.dandiarchive.org - redirector: null diff --git a/docs/source/cmdline/move.rst b/docs/source/cmdline/move.rst index c675eff6b..f6f0e0b9f 100644 --- a/docs/source/cmdline/move.rst +++ b/docs/source/cmdline/move.rst @@ -37,10 +37,11 @@ replaced with the replacement string, after expanding any backreferences. Options ------- -.. option:: -i, --dandi-instance +.. option:: -i, --dandi-instance - DANDI instance containing the remote Dandiset corresponding to the local - Dandiset in the current directory [default: ``dandi``] + DANDI instance (either a base URL or a known instance name) containing the + remote Dandiset corresponding to the local Dandiset in the current + directory [default: ``dandi``] .. option:: -d, --dandiset diff --git a/docs/source/cmdline/service-scripts.rst b/docs/source/cmdline/service-scripts.rst index 4d9d18537..bf37211bf 100644 --- a/docs/source/cmdline/service-scripts.rst +++ b/docs/source/cmdline/service-scripts.rst @@ -60,10 +60,10 @@ Options Specify the ID of the Dandiset to operate on. This option is required. -.. option:: -i, --dandi-instance +.. option:: -i, --dandi-instance - Specify the DANDI instance where the Dandiset is located [default: - ``dandi``] + DANDI instance (either a base URL or a known instance name) where the + Dandiset is located [default: ``dandi``] .. option:: -e, --existing [ask|overwrite|skip] diff --git a/docs/source/cmdline/upload.rst b/docs/source/cmdline/upload.rst index 2967cb927..ac8934226 100644 --- a/docs/source/cmdline/upload.rst +++ b/docs/source/cmdline/upload.rst @@ -34,9 +34,10 @@ Options - ``refresh`` [default] — upload only if local modification time is ahead of the remote -.. option:: -i, --dandi-instance +.. option:: -i, --dandi-instance - DANDI instance to upload to [default: ``dandi``] + DANDI instance (either a base URL or a known instance name) to upload to + [default: ``dandi``] .. option:: -J, --jobs N[:M] diff --git a/docs/source/ref/urls.rst b/docs/source/ref/urls.rst index fb448e106..7e1d75ddd 100644 --- a/docs/source/ref/urls.rst +++ b/docs/source/ref/urls.rst @@ -8,11 +8,10 @@ Resource Identifiers ``dandi`` commands and Python functions accept URLs and URL-like identifiers in the following formats for identifying Dandisets, assets, and asset collections. -Text in [brackets] is optional. A ``server`` field is a base API, GUI, or -redirector URL for a registered DANDI Archive instance. If an optional -``version`` field is omitted from a URL, the given Dandiset's most recent -published version will be used if it has one, and its draft version will be -used otherwise. +Text in [brackets] is optional. A ``server`` field is a base API or GUI URL +for a DANDI Archive instance. If an optional ``version`` field is omitted from +a URL, the given Dandiset's most recent published version will be used if it +has one, and its draft version will be used otherwise. - :samp:`https://identifiers.org/DANDI:{dandiset-id}[/{version}]` (case insensitive; ``version`` cannot be "draft") when it redirects @@ -22,7 +21,7 @@ used otherwise. — Refers to a Dandiset on the main archive instance named "dandi". `parse_dandi_url()` converts this format to a `DandisetURL`. -- Any ``https://dandiarchive.org/`` or +- Any ``https://gui.dandiarchive.org/`` or ``https://*dandiarchive-org.netflify.app/`` URL which redirects to one of the other URL formats diff --git a/tox.ini b/tox.ini index 410a56363..fe28b91be 100644 --- a/tox.ini +++ b/tox.ini @@ -47,7 +47,6 @@ addopts = --tb=short --durations=10 markers = integration obolibrary - redirector filterwarnings = error ignore:No cached namespaces found .*:UserWarning