From 0c4e859547cd522bb87ac85a0d313925f9123605 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sat, 30 May 2020 23:42:38 +0100 Subject: [PATCH 01/14] gdrive: add open Fixes #3408 Related #2865 Fixes #3897 --- dvc/remote/gdrive.py | 20 +++++++++++++++++++ dvc/utils/http.py | 46 ++------------------------------------------ dvc/utils/stream.py | 45 +++++++++++++++++++++++++++++++++++++++++++ setup.py | 2 +- 4 files changed, 68 insertions(+), 45 deletions(-) create mode 100644 dvc/utils/stream.py diff --git a/dvc/remote/gdrive.py b/dvc/remote/gdrive.py index 0204a8997e..c205373d18 100644 --- a/dvc/remote/gdrive.py +++ b/dvc/remote/gdrive.py @@ -1,9 +1,11 @@ +import io import logging import os import posixpath import re import threading from collections import defaultdict +from contextlib import contextmanager from urllib.parse import urlparse from funcy import cached_property, retry, wrap_prop, wrap_with @@ -15,6 +17,7 @@ from dvc.remote.base import BaseRemote, BaseRemoteTree from dvc.scheme import Schemes from dvc.utils import format_link, tmp_fname +from dvc.utils.stream import IterStream logger = logging.getLogger(__name__) FOLDER_MIME_TYPE = "application/vnd.google-apps.folder" @@ -389,6 +392,23 @@ def _gdrive_download_file( ) as pbar: gdrive_file.GetContentFile(to_file, callback=pbar.update_to) + @contextmanager + @_gdrive_retry + def open(self, path_info, mode="r", encoding=None): + assert mode in {"r", "rt", "rb"} + + item_id = self._get_item_id(path_info) + param = {"id": item_id} + # it does not create a file on the remote + gdrive_file = self._drive.CreateFile(param) + fd = gdrive_file.GetContentIOBuffer() + stream = IterStream(iter(fd)) + + if mode != "rb": + stream = io.TextIOWrapper(stream, encoding=encoding) + + yield stream + @_gdrive_retry def gdrive_delete_file(self, item_id): from pydrive2.files import ApiRequestError diff --git a/dvc/utils/http.py b/dvc/utils/http.py index b1fb13cb79..4472b80912 100644 --- a/dvc/utils/http.py +++ b/dvc/utils/http.py @@ -1,6 +1,8 @@ import io from contextlib import contextmanager +from dvc.utils.stream import IterStream + @contextmanager def open_url(url, mode="r", encoding=None): @@ -61,47 +63,3 @@ def gen(response): finally: # Ensure connection is closed it.close() - - -class IterStream(io.RawIOBase): - """Wraps an iterator yielding bytes as a file object""" - - def __init__(self, iterator): - self.iterator = iterator - self.leftover = None - - def readable(self): - return True - - # Python 3 requires only .readinto() method, it still uses other ones - # under some circumstances and falls back if those are absent. Since - # iterator already constructs byte strings for us, .readinto() is not the - # most optimal, so we provide .read1() too. - - def readinto(self, b): - try: - n = len(b) # We're supposed to return at most this much - chunk = self.leftover or next(self.iterator) - output, self.leftover = chunk[:n], chunk[n:] - - n_out = len(output) - b[:n_out] = output - return n_out - except StopIteration: - return 0 # indicate EOF - - readinto1 = readinto - - def read1(self, n=-1): - try: - chunk = self.leftover or next(self.iterator) - except StopIteration: - return b"" - - # Return an arbitrary number or bytes - if n <= 0: - self.leftover = None - return chunk - - output, self.leftover = chunk[:n], chunk[n:] - return output diff --git a/dvc/utils/stream.py b/dvc/utils/stream.py new file mode 100644 index 0000000000..6109475030 --- /dev/null +++ b/dvc/utils/stream.py @@ -0,0 +1,45 @@ +import io + + +class IterStream(io.RawIOBase): + """Wraps an iterator yielding bytes as a file object""" + + def __init__(self, iterator): + self.iterator = iterator + self.leftover = None + + def readable(self): + return True + + # Python 3 requires only .readinto() method, it still uses other ones + # under some circumstances and falls back if those are absent. Since + # iterator already constructs byte strings for us, .readinto() is not the + # most optimal, so we provide .read1() too. + + def readinto(self, b): + try: + n = len(b) # We're supposed to return at most this much + chunk = self.leftover or next(self.iterator) + output, self.leftover = chunk[:n], chunk[n:] + + n_out = len(output) + b[:n_out] = output + return n_out + except StopIteration: + return 0 # indicate EOF + + readinto1 = readinto + + def read1(self, n=-1): + try: + chunk = self.leftover or next(self.iterator) + except StopIteration: + return b"" + + # Return an arbitrary number or bytes + if n <= 0: + self.leftover = None + return chunk + + output, self.leftover = chunk[:n], chunk[n:] + return output diff --git a/setup.py b/setup.py index d60c928e01..02d3afdd62 100644 --- a/setup.py +++ b/setup.py @@ -84,7 +84,7 @@ def run(self): # Extra dependencies for remote integrations gs = ["google-cloud-storage==1.19.0"] -gdrive = ["pydrive2>=1.4.13"] +gdrive = ["pydrive2>=1.4.15"] s3 = ["boto3>=1.9.201"] azure = ["azure-storage-blob==2.1.0"] oss = ["oss2==2.6.1"] From 78dc1b971800cb0ca7e7809df05a4ee44d66556c Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sun, 31 May 2020 00:14:57 +0100 Subject: [PATCH 02/14] dependency: add gdrive --- dvc/dependency/__init__.py | 3 +++ dvc/dependency/gdrive.py | 7 +++++++ 2 files changed, 10 insertions(+) create mode 100644 dvc/dependency/gdrive.py diff --git a/dvc/dependency/__init__.py b/dvc/dependency/__init__.py index 1a778dbfc0..3edbc36ab3 100644 --- a/dvc/dependency/__init__.py +++ b/dvc/dependency/__init__.py @@ -3,6 +3,7 @@ import dvc.output as output from dvc.dependency.azure import AzureDependency +from dvc.dependency.gdrive import GDriveDependency from dvc.dependency.gs import GSDependency from dvc.dependency.hdfs import HDFSDependency from dvc.dependency.http import HTTPDependency @@ -19,6 +20,7 @@ DEPS = [ AzureDependency, + GDriveDependency, GSDependency, HDFSDependency, HTTPDependency, @@ -33,6 +35,7 @@ Schemes.SSH: SSHDependency, Schemes.S3: S3Dependency, Schemes.AZURE: AzureDependency, + Schemes.GDRIVE: GDriveDependency, Schemes.GS: GSDependency, Schemes.HDFS: HDFSDependency, Schemes.HTTP: HTTPDependency, diff --git a/dvc/dependency/gdrive.py b/dvc/dependency/gdrive.py new file mode 100644 index 0000000000..8bdbc6c744 --- /dev/null +++ b/dvc/dependency/gdrive.py @@ -0,0 +1,7 @@ +from dvc.dependency.base import BaseDependency +from dvc.output.base import BaseOutput +from dvc.remote.gdrive import GDriveRemote + + +class GDriveDependency(BaseDependency, BaseOutput): + REMOTE = GDriveRemote From 3cfdcc34b0a27a83f719c7a14aaa450cbf832b04 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sun, 31 May 2020 00:20:24 +0100 Subject: [PATCH 03/14] test: api: open: gdrive --- tests/func/test_api.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/func/test_api.py b/tests/func/test_api.py index 825ffa57a1..4f7b91ae08 100644 --- a/tests/func/test_api.py +++ b/tests/func/test_api.py @@ -8,7 +8,7 @@ from dvc.main import main from dvc.path_info import URLInfo from dvc.utils.fs import remove -from tests.remotes import GCP, HDFS, OSS, S3, SSH, Azure, Local +from tests.remotes import GCP, HDFS, OSS, S3, SSH, Azure, GDrive, Local remote_params = [S3, GCP, Azure, OSS, SSH, HDFS] all_remote_params = [Local] + remote_params @@ -56,7 +56,9 @@ def test_get_url_requires_dvc(tmp_dir, scm): api.get_url("foo", repo=f"file://{tmp_dir}") -@pytest.mark.parametrize("remote_url", all_remote_params, indirect=True) +@pytest.mark.parametrize( + "remote_url", all_remote_params + [GDrive], indirect=True +) def test_open(remote_url, tmp_dir, dvc): run_dvc("remote", "add", "-d", "upstream", remote_url) tmp_dir.dvc_gen("foo", "foo-text") From 2a140c92504b89f0c9959a32cf330bff12a33711 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sun, 31 May 2020 00:51:05 +0100 Subject: [PATCH 04/14] rollback pydrive dependency --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 02d3afdd62..87dad025b7 100644 --- a/setup.py +++ b/setup.py @@ -84,7 +84,7 @@ def run(self): # Extra dependencies for remote integrations gs = ["google-cloud-storage==1.19.0"] -gdrive = ["pydrive2>=1.4.15"] +gdrive = ["pydrive2>=1.4.14"] s3 = ["boto3>=1.9.201"] azure = ["azure-storage-blob==2.1.0"] oss = ["oss2==2.6.1"] From 405684d4465045a5ccdb243640d22097d4b4d068 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sun, 31 May 2020 16:12:11 +0100 Subject: [PATCH 05/14] Revert "dependency: add gdrive" This reverts commit fd333261d2e9fe1364cc278138c79890a1f5fe7a. --- dvc/dependency/__init__.py | 3 --- dvc/dependency/gdrive.py | 7 ------- 2 files changed, 10 deletions(-) delete mode 100644 dvc/dependency/gdrive.py diff --git a/dvc/dependency/__init__.py b/dvc/dependency/__init__.py index 3edbc36ab3..1a778dbfc0 100644 --- a/dvc/dependency/__init__.py +++ b/dvc/dependency/__init__.py @@ -3,7 +3,6 @@ import dvc.output as output from dvc.dependency.azure import AzureDependency -from dvc.dependency.gdrive import GDriveDependency from dvc.dependency.gs import GSDependency from dvc.dependency.hdfs import HDFSDependency from dvc.dependency.http import HTTPDependency @@ -20,7 +19,6 @@ DEPS = [ AzureDependency, - GDriveDependency, GSDependency, HDFSDependency, HTTPDependency, @@ -35,7 +33,6 @@ Schemes.SSH: SSHDependency, Schemes.S3: S3Dependency, Schemes.AZURE: AzureDependency, - Schemes.GDRIVE: GDriveDependency, Schemes.GS: GSDependency, Schemes.HDFS: HDFSDependency, Schemes.HTTP: HTTPDependency, diff --git a/dvc/dependency/gdrive.py b/dvc/dependency/gdrive.py deleted file mode 100644 index 8bdbc6c744..0000000000 --- a/dvc/dependency/gdrive.py +++ /dev/null @@ -1,7 +0,0 @@ -from dvc.dependency.base import BaseDependency -from dvc.output.base import BaseOutput -from dvc.remote.gdrive import GDriveRemote - - -class GDriveDependency(BaseDependency, BaseOutput): - REMOTE = GDriveRemote From 802577ff12186bcec8a7b14b27363c84f8fe77c8 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sun, 31 May 2020 20:26:44 +0100 Subject: [PATCH 06/14] tests: remotes: GDrive: fix get_url --- tests/remotes.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/remotes.py b/tests/remotes.py index a5ce43a30a..cff71a173f 100644 --- a/tests/remotes.py +++ b/tests/remotes.py @@ -22,6 +22,7 @@ TEST_AWS_REPO_BUCKET = os.environ.get("DVC_TEST_AWS_REPO_BUCKET", "dvc-temp") TEST_GCP_REPO_BUCKET = os.environ.get("DVC_TEST_GCP_REPO_BUCKET", "dvc-test") +TEST_GDRIVE_REPO_BUCKET = "root" TEST_OSS_REPO_BUCKET = "dvc-test" TEST_GCP_CREDS_FILE = os.path.abspath( @@ -152,10 +153,14 @@ def create_dir(dvc, url): remote = GDriveRemote(dvc, config) remote.tree._gdrive_create_dir("root", remote.path_info.path) + @staticmethod + def get_storagepath(): + return TEST_GDRIVE_REPO_BUCKET + "/" + str(uuid.uuid4()) + @staticmethod def get_url(): # NOTE: `get_url` should always return new random url - return "gdrive://root/" + str(uuid.uuid4()) + return "gdrive://" + GDrive.get_storagepath() class Azure: From 61c112580f765e1298359681c67f5c383e187c6b Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sun, 31 May 2020 20:31:39 +0100 Subject: [PATCH 07/14] tests: api: fully test GDrive --- tests/func/test_api.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/func/test_api.py b/tests/func/test_api.py index 4f7b91ae08..068a0fdb20 100644 --- a/tests/func/test_api.py +++ b/tests/func/test_api.py @@ -10,7 +10,7 @@ from dvc.utils.fs import remove from tests.remotes import GCP, HDFS, OSS, S3, SSH, Azure, GDrive, Local -remote_params = [S3, GCP, Azure, OSS, SSH, HDFS] +remote_params = [S3, GCP, Azure, GDrive, OSS, SSH, HDFS] all_remote_params = [Local] + remote_params @@ -56,9 +56,7 @@ def test_get_url_requires_dvc(tmp_dir, scm): api.get_url("foo", repo=f"file://{tmp_dir}") -@pytest.mark.parametrize( - "remote_url", all_remote_params + [GDrive], indirect=True -) +@pytest.mark.parametrize("remote_url", all_remote_params, indirect=True) def test_open(remote_url, tmp_dir, dvc): run_dvc("remote", "add", "-d", "upstream", remote_url) tmp_dir.dvc_gen("foo", "foo-text") From a6f93c015f6b454ad0bb8ebe3119bac691ecd97f Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sun, 31 May 2020 21:01:23 +0100 Subject: [PATCH 08/14] minor typo fix --- dvc/remote/gdrive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/remote/gdrive.py b/dvc/remote/gdrive.py index c205373d18..017a3fcc2c 100644 --- a/dvc/remote/gdrive.py +++ b/dvc/remote/gdrive.py @@ -35,7 +35,7 @@ def __init__(self, cred_location): if cred_location: message = ( "GDrive remote auth failed with credentials in '{}'.\n" - "Backup first, remove of fix them, and run DVC again.\n" + "Backup first, remove or fix them, and run DVC again.\n" "It should do auth again and refresh the credentials.\n\n" "Details:".format(cred_location) ) From 7285161d1be7a44c2b9bdeaded03a2facf3f713c Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Fri, 12 Jun 2020 21:48:08 +0100 Subject: [PATCH 09/14] attempt gdrive tests fix --- tests/func/test_api.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/func/test_api.py b/tests/func/test_api.py index 068a0fdb20..010cde444a 100644 --- a/tests/func/test_api.py +++ b/tests/func/test_api.py @@ -21,6 +21,11 @@ def remote_url(request): return request.param.get_url() +def ensure_dir(dvc, url): + if url.startswith("gdrive://"): + GDrive.create_dir(dvc, url) + + def run_dvc(*argv): assert main(argv) == 0 @@ -58,6 +63,7 @@ def test_get_url_requires_dvc(tmp_dir, scm): @pytest.mark.parametrize("remote_url", all_remote_params, indirect=True) def test_open(remote_url, tmp_dir, dvc): + ensure_dir(dvc, remote_url) run_dvc("remote", "add", "-d", "upstream", remote_url) tmp_dir.dvc_gen("foo", "foo-text") run_dvc("push") @@ -71,6 +77,7 @@ def test_open(remote_url, tmp_dir, dvc): @pytest.mark.parametrize("remote_url", all_remote_params, indirect=True) def test_open_external(remote_url, erepo_dir, setup_remote): + ensure_dir(erepo_dir.dvc, remote_url) setup_remote(erepo_dir.dvc, url=remote_url) with erepo_dir.chdir(): @@ -95,6 +102,7 @@ def test_open_external(remote_url, erepo_dir, setup_remote): @pytest.mark.parametrize("remote_url", all_remote_params, indirect=True) def test_open_granular(remote_url, tmp_dir, dvc): + ensure_dir(dvc, remote_url) run_dvc("remote", "add", "-d", "upstream", remote_url) tmp_dir.dvc_gen({"dir": {"foo": "foo-text"}}) run_dvc("push") @@ -108,6 +116,7 @@ def test_open_granular(remote_url, tmp_dir, dvc): @pytest.mark.parametrize("remote_url", all_remote_params, indirect=True) def test_missing(remote_url, tmp_dir, dvc): + ensure_dir(dvc, remote_url) tmp_dir.dvc_gen("foo", "foo") run_dvc("remote", "add", "-d", "upstream", remote_url) From 5aee39ad195b594048a60de4b811d1299746a769 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sat, 13 Jun 2020 14:19:26 +0100 Subject: [PATCH 10/14] fix gdrive credentials config --- tests/func/test_api.py | 53 +++++++++++++++++++++++++++++++++--------- 1 file changed, 42 insertions(+), 11 deletions(-) diff --git a/tests/func/test_api.py b/tests/func/test_api.py index 010cde444a..d5433dc164 100644 --- a/tests/func/test_api.py +++ b/tests/func/test_api.py @@ -8,7 +8,17 @@ from dvc.main import main from dvc.path_info import URLInfo from dvc.utils.fs import remove -from tests.remotes import GCP, HDFS, OSS, S3, SSH, Azure, GDrive, Local +from tests.remotes import ( + GCP, + HDFS, + OSS, + S3, + SSH, + TEST_REMOTE, + Azure, + GDrive, + Local, +) remote_params = [S3, GCP, Azure, GDrive, OSS, SSH, HDFS] all_remote_params = [Local] + remote_params @@ -21,18 +31,39 @@ def remote_url(request): return request.param.get_url() +def run_dvc(*argv): + assert main(argv) == 0 + + def ensure_dir(dvc, url): if url.startswith("gdrive://"): GDrive.create_dir(dvc, url) - - -def run_dvc(*argv): - assert main(argv) == 0 + run_dvc( + "remote", + "modify", + TEST_REMOTE, + "gdrive_service_account_email", + "test", + ) + run_dvc( + "remote", + "modify", + TEST_REMOTE, + "gdrive_service_account_p12_file_path", + "test.p12", + ) + run_dvc( + "remote", + "modify", + TEST_REMOTE, + "gdrive_use_service_account", + "True", + ) @pytest.mark.parametrize("remote_url", remote_params, indirect=True) def test_get_url(tmp_dir, dvc, remote_url): - run_dvc("remote", "add", "-d", "upstream", remote_url) + run_dvc("remote", "add", "-d", TEST_REMOTE, remote_url) tmp_dir.dvc_gen("foo", "foo") expected_url = URLInfo(remote_url) / "ac/bd18db4cc2f85cedef654fccc4a4d8" @@ -63,8 +94,8 @@ def test_get_url_requires_dvc(tmp_dir, scm): @pytest.mark.parametrize("remote_url", all_remote_params, indirect=True) def test_open(remote_url, tmp_dir, dvc): + run_dvc("remote", "add", "-d", TEST_REMOTE, remote_url) ensure_dir(dvc, remote_url) - run_dvc("remote", "add", "-d", "upstream", remote_url) tmp_dir.dvc_gen("foo", "foo-text") run_dvc("push") @@ -77,8 +108,8 @@ def test_open(remote_url, tmp_dir, dvc): @pytest.mark.parametrize("remote_url", all_remote_params, indirect=True) def test_open_external(remote_url, erepo_dir, setup_remote): - ensure_dir(erepo_dir.dvc, remote_url) setup_remote(erepo_dir.dvc, url=remote_url) + ensure_dir(erepo_dir.dvc, remote_url) with erepo_dir.chdir(): erepo_dir.dvc_gen("version", "master", commit="add version") @@ -102,8 +133,8 @@ def test_open_external(remote_url, erepo_dir, setup_remote): @pytest.mark.parametrize("remote_url", all_remote_params, indirect=True) def test_open_granular(remote_url, tmp_dir, dvc): + run_dvc("remote", "add", "-d", TEST_REMOTE, remote_url) ensure_dir(dvc, remote_url) - run_dvc("remote", "add", "-d", "upstream", remote_url) tmp_dir.dvc_gen({"dir": {"foo": "foo-text"}}) run_dvc("push") @@ -116,9 +147,9 @@ def test_open_granular(remote_url, tmp_dir, dvc): @pytest.mark.parametrize("remote_url", all_remote_params, indirect=True) def test_missing(remote_url, tmp_dir, dvc): - ensure_dir(dvc, remote_url) tmp_dir.dvc_gen("foo", "foo") - run_dvc("remote", "add", "-d", "upstream", remote_url) + run_dvc("remote", "add", "-d", TEST_REMOTE, remote_url) + ensure_dir(dvc, remote_url) # Remove cache to make foo missing remove(dvc.cache.local.cache_dir) From ba52fc130db3b6556edb43ea5746e51c721f1a2f Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sat, 13 Jun 2020 18:16:29 +0100 Subject: [PATCH 11/14] replace GDrivePathNotFound => FileMissingError --- dvc/exceptions.py | 7 +++++-- dvc/remote/gdrive.py | 12 +++--------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/dvc/exceptions.py b/dvc/exceptions.py index 32545c5f75..d59972db87 100644 --- a/dvc/exceptions.py +++ b/dvc/exceptions.py @@ -222,9 +222,12 @@ def __init__(self, etag, cached_etag): class FileMissingError(DvcException): - def __init__(self, path): + def __init__(self, path, hint=None): self.path = path - super().__init__(f"Can't find '{path}' neither locally nor on remote") + self.hint = "" if hint is None else f". {hint}" + super().__init__( + f"Can't find '{path}' neither locally nor on remote{hint}" + ) class DvcIgnoreInCollectedDirError(DvcException): diff --git a/dvc/remote/gdrive.py b/dvc/remote/gdrive.py index 017a3fcc2c..d983c1eed9 100644 --- a/dvc/remote/gdrive.py +++ b/dvc/remote/gdrive.py @@ -11,7 +11,7 @@ from funcy import cached_property, retry, wrap_prop, wrap_with from funcy.py3 import cat -from dvc.exceptions import DvcException +from dvc.exceptions import DvcException, FileMissingError from dvc.path_info import CloudURLInfo from dvc.progress import Tqdm from dvc.remote.base import BaseRemote, BaseRemoteTree @@ -23,12 +23,6 @@ FOLDER_MIME_TYPE = "application/vnd.google-apps.folder" -class GDrivePathNotFound(DvcException): - def __init__(self, path_info, hint): - hint = "" if hint is None else f" {hint}" - super().__init__(f"GDrive path '{path_info}' not found.{hint}") - - class GDriveAuthError(DvcException): def __init__(self, cred_location): @@ -522,12 +516,12 @@ def _get_item_id(self, path_info, create=False, use_cache=True, hint=None): return min(item_ids) assert not create - raise GDrivePathNotFound(path_info, hint) + raise FileMissingError(path_info, hint) def exists(self, path_info): try: self._get_item_id(path_info) - except GDrivePathNotFound: + except FileMissingError: return False else: return True From a84311a858b82f5b4826a43f4d981f19ed3308ea Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sat, 13 Jun 2020 18:16:56 +0100 Subject: [PATCH 12/14] fix test_open_external[GDrive] --- tests/func/test_api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/func/test_api.py b/tests/func/test_api.py index d5433dc164..cab427faa5 100644 --- a/tests/func/test_api.py +++ b/tests/func/test_api.py @@ -109,6 +109,7 @@ def test_open(remote_url, tmp_dir, dvc): @pytest.mark.parametrize("remote_url", all_remote_params, indirect=True) def test_open_external(remote_url, erepo_dir, setup_remote): setup_remote(erepo_dir.dvc, url=remote_url) + run_dvc("remote", "add", TEST_REMOTE, remote_url) ensure_dir(erepo_dir.dvc, remote_url) with erepo_dir.chdir(): From 8125a6540340d7c2a51f3139488b2882095155ca Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sat, 13 Jun 2020 19:06:49 +0100 Subject: [PATCH 13/14] add ensure_dir_scm --- tests/func/test_api.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/func/test_api.py b/tests/func/test_api.py index cab427faa5..6e69d35831 100644 --- a/tests/func/test_api.py +++ b/tests/func/test_api.py @@ -61,6 +61,19 @@ def ensure_dir(dvc, url): ) +def ensure_dir_scm(dvc, url): + if url.startswith("gdrive://"): + GDrive.create_dir(dvc, url) + with dvc.config.edit() as conf: + conf["remote"][TEST_REMOTE].update( + gdrive_service_account_email="test", + gdrive_service_account_p12_file_path="test.p12", + gdrive_use_service_account=True, + ) + dvc.scm.add(dvc.config.files["repo"]) + dvc.scm.commit(f"modify '{TEST_REMOTE}' remote") + + @pytest.mark.parametrize("remote_url", remote_params, indirect=True) def test_get_url(tmp_dir, dvc, remote_url): run_dvc("remote", "add", "-d", TEST_REMOTE, remote_url) @@ -109,8 +122,7 @@ def test_open(remote_url, tmp_dir, dvc): @pytest.mark.parametrize("remote_url", all_remote_params, indirect=True) def test_open_external(remote_url, erepo_dir, setup_remote): setup_remote(erepo_dir.dvc, url=remote_url) - run_dvc("remote", "add", TEST_REMOTE, remote_url) - ensure_dir(erepo_dir.dvc, remote_url) + ensure_dir_scm(erepo_dir.dvc, remote_url) with erepo_dir.chdir(): erepo_dir.dvc_gen("version", "master", commit="add version") From 8ae3016c031b80a058fdfa6ef58b6015162bfdab Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sat, 13 Jun 2020 20:15:04 +0100 Subject: [PATCH 14/14] minor exception fix --- dvc/exceptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/exceptions.py b/dvc/exceptions.py index d59972db87..a886bb965e 100644 --- a/dvc/exceptions.py +++ b/dvc/exceptions.py @@ -224,7 +224,7 @@ def __init__(self, etag, cached_etag): class FileMissingError(DvcException): def __init__(self, path, hint=None): self.path = path - self.hint = "" if hint is None else f". {hint}" + hint = "" if hint is None else f". {hint}" super().__init__( f"Can't find '{path}' neither locally nor on remote{hint}" )