From 69c1eacdcbc03da36a8566b1f2311f11c3163ce4 Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Tue, 29 Oct 2019 17:29:58 -0700 Subject: [PATCH 1/3] pull: fix-2684 - failing while pulling imported directories --- dvc/output/base.py | 2 +- dvc/repo/fetch.py | 34 +++++++++++++++++----------------- tests/func/test_import.py | 18 ++++++++++++++++++ 3 files changed, 36 insertions(+), 18 deletions(-) diff --git a/dvc/output/base.py b/dvc/output/base.py index f940d3f52e..8fa3ad367c 100644 --- a/dvc/output/base.py +++ b/dvc/output/base.py @@ -334,7 +334,7 @@ def unprotect(self): self.remote.unprotect(self.path_info) def _collect_used_dir_cache(self, remote=None, force=False, jobs=None): - """Get a list of `info`s retaled to the given directory. + """Get a list of `info`s related to the given directory. - Pull the directory entry from the remote cache if it was changed. diff --git a/dvc/repo/fetch.py b/dvc/repo/fetch.py index 7029461630..cb0cd15736 100644 --- a/dvc/repo/fetch.py +++ b/dvc/repo/fetch.py @@ -76,24 +76,24 @@ def _fetch_external(self, repo_url, repo_rev, files): cache_dir = self.cache.local.cache_dir try: with external_repo(repo_url, repo_rev, cache_dir=cache_dir) as repo: - cache = NamedCache() - for name in files: - try: - out = repo.find_out_by_relpath(name) - except OutputNotFoundError: - failed += 1 - logger.exception( - "failed to fetch data for '{}'".format(name) - ) - continue - else: - cache.update(out.get_used_cache()) - with repo.state: - try: - return repo.cloud.pull(cache), failed - except DownloadError as exc: - failed += exc.amount + cache = NamedCache() + for name in files: + try: + out = repo.find_out_by_relpath(name) + except OutputNotFoundError: + failed += 1 + logger.exception( + "failed to fetch data for '{}'".format(name) + ) + continue + else: + cache.update(out.get_used_cache()) + + try: + return repo.cloud.pull(cache), failed + except DownloadError as exc: + failed += exc.amount except CloneError: failed += 1 logger.exception( diff --git a/tests/func/test_import.py b/tests/func/test_import.py index fc00ee7b1e..5e7313d18e 100644 --- a/tests/func/test_import.py +++ b/tests/func/test_import.py @@ -2,6 +2,7 @@ import os import filecmp +import shutil import pytest from mock import patch @@ -67,6 +68,23 @@ def test_pull_imported_stage(dvc_repo, erepo): assert os.path.isfile(dst_cache) +def test_pull_imported_directory_stage(dvc_repo, erepo): + src = erepo.DATA_DIR + dst = erepo.DATA_DIR + "_imported" + stage_file = erepo.DATA_DIR + "_imported.dvc" + + dvc_repo.imp(erepo.root_dir, src, dst) + + shutil.rmtree(dst) + shutil.rmtree(dvc_repo.cache.local.cache_dir) + + dvc_repo.pull([stage_file]) + + assert os.path.exists(dst) + assert os.path.isdir(dst) + trees_equal(src, dst) + + def test_download_error_pulling_imported_stage(dvc_repo, erepo): src = erepo.FOO dst = erepo.FOO + "_imported" From 85f43340b996c9042cfa7bac5fc82002445806f8 Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Wed, 30 Oct 2019 07:30:47 -0700 Subject: [PATCH 2/3] pull: fix-2684 - fix code indentation --- dvc/repo/fetch.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dvc/repo/fetch.py b/dvc/repo/fetch.py index cb0cd15736..b69c8c04b2 100644 --- a/dvc/repo/fetch.py +++ b/dvc/repo/fetch.py @@ -90,10 +90,10 @@ def _fetch_external(self, repo_url, repo_rev, files): else: cache.update(out.get_used_cache()) - try: - return repo.cloud.pull(cache), failed - except DownloadError as exc: - failed += exc.amount + try: + return repo.cloud.pull(cache), failed + except DownloadError as exc: + failed += exc.amount except CloneError: failed += 1 logger.exception( From 747e5c9e30def9e39772f7481ae41a3b5d2dd677 Mon Sep 17 00:00:00 2001 From: Ivan Shcheklein Date: Wed, 30 Oct 2019 07:47:41 -0700 Subject: [PATCH 3/3] minor imported directories pull test update Co-Authored-By: Ruslan Kuprieiev --- tests/func/test_import.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/func/test_import.py b/tests/func/test_import.py index 5e7313d18e..4fe8857aad 100644 --- a/tests/func/test_import.py +++ b/tests/func/test_import.py @@ -71,7 +71,7 @@ def test_pull_imported_stage(dvc_repo, erepo): def test_pull_imported_directory_stage(dvc_repo, erepo): src = erepo.DATA_DIR dst = erepo.DATA_DIR + "_imported" - stage_file = erepo.DATA_DIR + "_imported.dvc" + stage_file = dst + ".dvc" dvc_repo.imp(erepo.root_dir, src, dst)