[TEST PR] debugging tests failure for subrepo support

.travis.yml

@@ -10,9 +10,7 @@ env:
     - PATH="$PYENV_ROOT/bin:$PATH"
     - PIP_CACHE_DIR="$HOME/.cache/pip"  # unify pip cache location for all platforms
 stages:
-  - check
   - test
-  - build
 # Travis doesn't support testing python on Mac yet, so we need
 # to workaround it by installing it directly with brew.
 #
@@ -24,93 +22,10 @@ stages:
 jobs:
   include:
   # patch formatting checks
-  - name: "Check patch formatting"
-    stage: check
-    os: linux
-    language: python
-    python: 3.7
-    before_install: bash ./scripts/ci/install.sh
-    install:
-    script: ./scripts/ci/check_patch.sh
   # test jobs
-  - name: "3.7 on Windows"
-    stage: test
-    os: windows           # Windows 10.0.17134 N/A Build 17134
-    language: shell       # 'language: python' is an error on Travis CI Windows
-    env:
-      - PYTHON_VERSION=3.7.5
-      - PATH=/c/Python37:/c/Python37/Scripts:$PATH
-  - name: "3.8 on Windows"
-    os: windows           # Windows 10.0.17134 N/A Build 17134
-    language: shell       # 'language: python' is an error on Travis CI Windows
-    env:
-      - PYTHON_VERSION=3.8.0
-      - PATH=/c/Python38:/c/Python38/Scripts:$PATH
-  - name: "3.6 on Linux"
-    language: python
-    python: 3.6
-  - name: "3.7 on Linux"
-    language: python
-    python: 3.7
-  - name: "3.8 on Linux"
-    language: python
-    python: 3.8
   - name: "3.7 on Mac"
     os: osx
     osx_image: xcode11.3
-  # build jobs
-  - name: "Mac pkg"
-    stage: build
-    os: osx
-    osx_image: xcode9.4
-    language: ruby
-    rvm: 2.4.3
-    install:
-    script: ./scripts/build_posix.sh
-  - name: "Windows pkg"
-    os: windows
-    language: shell
-    install:
-    script: ./scripts/build_windows.cmd
-    env:
-      - PYTHON_VERSION=3.7.5
-      - PATH=/c/Python37:/c/Python37/Scripts:$PATH
-  - name: "Linux pkgs"
-    language: python
-    python: 3.7
-    before_install:
-    install:
-    script: ./scripts/build_posix.sh
-  - name: "PyPI pkgs"
-    language: python
-    python: 3.8
-    script: ./scripts/build_package.sh
-  - name: Snapcraft snap
-    language: python
-    python: 3.8
-    addons:
-     snaps:
-     - name: snapcraft
-       channel: stable
-       confinement: classic
-     - name: review-tools
-       channel: stable
-     - name: lxd
-       channel: stable
-    env:
-    - SNAPCRAFT_IMAGE_INFO: |
-        '{"build_url": "$TRAVIS_JOB_WEB_URL"}'
-    - SNAPCRAFT_BUILD_ENVIRONMENT: lxd
-    - SNAPCRAFT_BUILD_INFO: 1  # https://snapcraft.io/blog/introducing-developer-notifications-for-snap-security-updates
-    install:
-    - sudo usermod --append --groups lxd $USER
-    - sudo /snap/bin/lxd.migrate -yes
-    - sudo /snap/bin/lxd waitready
-    - sudo /snap/bin/lxd init --auto
-    script:
-    - ./scripts/build_snap.sh
-    after_failure:
-    - sudo journalctl -u snapd
 
 before_install:
   - bash ./scripts/ci/before_install.sh
@@ -120,43 +35,3 @@ install:
   - bash ./scripts/ci/install.sh
 script:
   - bash ./scripts/ci/script.sh
-deploy:
-  - provider: releases
-    api_key:
-      secure: nyo17gF4Xl6DFL/KVkMewC805Cv+eK99L7SiYCU+WUAYDIzzFL3YJm7fm8bznPeor7DmpO7uSLSveMnJETBtdurW6JWhCTUss4yD66yFzpk6Z5cvHf/1UA5pdMHkkMM54fRaTRt6L2Yal7RPwGEQTBECLXW1Xmf78Y6+0l8lTvXQ9jXOQSRO/63VDKI5Zww2+XytVicUFlr0hLMGKUuaCjjXcQlAPzBgQz3/5Iz6EtzeTf2o+gTPjocbRbSFPikfiBXUn8XlJ8jBj7i114XJRJffw59KxYGDzxp+enfLBULGNoW0Np2hZqkKkPUIdB08C8XVbRXlFYX8cEYBczjqhv8A05p5F5CnltnqQ0KazcoUP2awPVqn68dWNRF+kqrnMBus8euYKIZ4dy/6bVEKJRa84ESJXZVDcwXTAXRqyovSxOlXqg9ZgAImMKBfKn/fpDX2xIvQPaXcnJxKbmcePfdbC1d0HYJ5jkbNaxKE/T15VvhzIho7mJ2WAv1r8QouQUuzqtpzIGbRn/KXHRUzCtyP0k4+eMKMLFOPzg848WsDarP3v8VYiSy4nFY7ZP7DRd3KttvW5bgVbSeO4jPmTgS6TnnpRwual9BJXSG7bjbfpIMii3bNZ7jWL4zpKT9Jfm8w9RGKLcX9HqkZ3+y60+LtYdhEfBvqsIlWQoIJBXg=
-    file_glob: true
-    file:
-      - dvc*.rpm
-      - dvc*.deb
-      - dvc*.pkg
-      - dvc*.exe
-    skip_cleanup: true
-    on:
-      condition: "$(./scripts/ci/deploy_condition.sh dvc*.rpm dvc*.deb dvc*.pkg dvc*.exe)"
-      tags: true
-      repo: iterative/dvc
-      stage: build
-  - provider: pypi
-    user:
-      secure: "IOzNfgPoG3ICg3+s4AuZCCnawgd/KqMzeiAB9R0B9Xb6ProPQJp9O3GBZrtg5OJMHdWQQtwULeOh/LWk4sFdXMcX1iNEgm5Tb7DwO7TDtltW14jUgN9AqfY2twj4UIsW/ON8w0P8v36ZoQq1kO6Pfgjijdm3w+9h6rpps8rJ6jXYpVZVO+SSzzijWddanMkEz4GgctZ2r7U5MWlmm1UogF60kEZIZCQykz0yJnhTpP9nMFDKw6Maki/mcfVwTQ/v3RqjzttHei/aaEMGQXYr3TOX8NPYJXerBpjShDZqcgcCfqPzhi7uCwVYaPvRsMMYH5jgtMz/SO3J6ToeGKT0f1F1ihRyCdusg325TMK7Vv8yHJzL4ngeUbb/oqxWtxhLY7f77Lkx6hBwe6ROpTqiQHIfZad5qyIHihKcDcIHo4FwLHVzxR4y+Ds2js5dXwBkUBaEAx0Dmc6ZdgXqxd815GDwv+VVpzXJhtO6qXmKE36F/Oe6eCNG3dk5MdOK4C2DrM9M8MXllqeQtX6JrM2stWlBalAc9Jn4b0xxAoghX3x3G3hNsLc9Uist1VinY2+IcPozusqj66c+Vbi5YngWSbICOjlIcq2GaOlBi3xioGts396442ztf1eLjMVuvykCI3b4Yz/vJYZOZjUvG/6nLfaFQTJLSXkTvI4do3foeY4="
-    password:
-      secure: "ts/3FSPL+w4euXryip9wCJWUqUwuJ2CqHi0nEpYVM10KL0cp9aMkQXN/7C8ih+j/JZ25kaO+tis1nx0JERVnKrg/1F/fJhxHTpakCP4OSIlhKRXDt7yUUHcKKt5QdM2XCKhApptP6RTpWL1vlc36CMrhdvTcMYp7hlDOiWjdhhim24FGQzabL42Ko59ttyyCROqgcnjHqj+WlhJl7iYO+lv2PLOtslpkIYY5EMjgBLuGyOvxTqZsAXtworsASdB2CrsTE6iMRcrM1afgkyBlUqDezoiv9TwfZrZ85oRoAxPB7su4QWK7lEP0wGQV/SDz/RL7c+cNfRaauxUKVcC8I2LfaEpudv8vi205+v/08wzPbS4xSOnf89irUJnyGxUy+bYLoXPPVPMyzcQRAUnrQl+ENCmN62HU+/2pkIyiUJvt0wYYP0vp+3pIbXbYQGmAIMq7nZJd7K3rikfnQIGKI03xZsFe3PEdjzhDI9Ej/tRUbJx4pgg084QuYYca6MSYn/zJ5pqIG4A7ngCSxoxoWZT0duo8PrUWQeT17L73Q86HeEv3OzZhWfEF2fNDN3Qbsu4CllSZnbE7DXgAuysTwTYHS/na6lp7wejOqdRUXrHbidVla9Mwxyc9JdWNWzUDxALvNxAdNbV6l85mO8izy8wjQF6M1O7idqTfjQeR4pc="
-    skip_existing: true
-    file_glob: true
-    file:
-      - dist/dvc*.whl
-      - dist/dvc*.tar.gz
-    skip_cleanup: true
-    on:
-      condition: "$(./scripts/ci/deploy_condition.sh dist/dvc*.whl dist/dvc*.tar.gz)"
-      tags: true
-      repo: iterative/dvc
-      stage: build
-  - provider: script
-    skip_cleanup: true
-    script: "(echo $SNAP_TOKEN | snapcraft login --with -) && timeout 600 snapcraft push dvc_*.snap --release $SNAP_CHANNEL || echo timed out"
-    on:
-      all_branches: true
-      condition: "$(./scripts/ci/deploy_condition.sh dvc_*.snap) && ($TRAVIS_BRANCH = master || -n $TRAVIS_TAG) && -n $SNAP_CHANNEL"
-      repo: iterative/dvc
-      stage: build

dvc/api.py

@@ -2,7 +2,12 @@
 from contextlib import _GeneratorContextManager as GCM
 from contextlib import contextmanager
 
-from dvc.exceptions import DvcException, NotDvcRepoError
+from dvc.exceptions import (
+    DvcException,
+    FileMissingError,
+    NotDvcRepoError,
+    PathMissingError,
+)
 from dvc.external_repo import external_repo
 from dvc.repo import Repo
 
@@ -26,10 +31,14 @@ def get_url(path, repo=None, rev=None, remote=None):
     directory in the remote storage.
     """
     with _make_repo(repo, rev=rev) as _repo:
-        if not isinstance(_repo, Repo):
-            raise UrlNotDvcRepoError(_repo.url)  # pylint: disable=no-member
-        out = _repo.find_out_by_relpath(path)
-        remote_obj = _repo.cloud.get_remote(remote)
+        # pylint: disable=no-member
+        path = os.path.join(_repo.root_dir, path)
+        is_erepo = not isinstance(_repo, Repo)
+        r = _repo.in_repo(path) if is_erepo else _repo
+        if is_erepo and not r:
+            raise UrlNotDvcRepoError(_repo.url)
+        out = r.find_out_by_relpath(path)
+        remote_obj = r.cloud.get_remote(remote)
         return str(remote_obj.tree.hash_to_path_info(out.checksum))
 
 
@@ -74,10 +83,17 @@ def __getattr__(self, name):
 
 def _open(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
     with _make_repo(repo, rev=rev) as _repo:
-        with _repo.open_by_relpath(
-            path, remote=remote, mode=mode, encoding=encoding
-        ) as fd:
-            yield fd
+        is_erepo = not isinstance(_repo, Repo)
+        try:
+            with _repo.repo_tree.open_by_relpath(
+                path, remote=remote, mode=mode, encoding=encoding
+            ) as fd:
+                yield fd
+        except FileNotFoundError as exc:
+            if is_erepo:
+                # pylint: disable=no-member
+                raise PathMissingError(path, _repo.url) from exc
+            raise FileMissingError(path) from exc
 
 
 def read(path, repo=None, rev=None, remote=None, mode="r", encoding=None):
@@ -101,5 +117,5 @@ def _make_repo(repo_url=None, rev=None):
             return
         except NotDvcRepoError:
             pass  # fallthrough to external_repo
-    with external_repo(url=repo_url, rev=rev) as repo:
+    with external_repo(url=repo_url, rev=rev, stream=True) as repo:
         yield repo

dvc/dependency/repo.py

@@ -2,7 +2,6 @@
 
 from voluptuous import Required
 
-from dvc.exceptions import OutputNotFoundError
 from dvc.path_info import PathInfo
 
 from .local import LocalDependency
@@ -42,30 +41,17 @@ def repo_pair(self):
     def __str__(self):
         return "{} ({})".format(self.def_path, self.def_repo[self.PARAM_URL])
 
-    def _make_repo(self, *, locked=True):
+    def _make_repo(self, *, locked=True, **kwargs):
         from dvc.external_repo import external_repo
 
         d = self.def_repo
         rev = (d.get("rev_lock") if locked else None) or d.get("rev")
-        return external_repo(d["url"], rev=rev)
+        return external_repo(d["url"], rev=rev, **kwargs)
 
     def _get_checksum(self, locked=True):
-        from dvc.repo.tree import RepoTree
-
-        with self._make_repo(locked=locked) as repo:
-            try:
-                return repo.find_out_by_relpath(self.def_path).info["md5"]
-            except OutputNotFoundError:
-                path = PathInfo(os.path.join(repo.root_dir, self.def_path))
-
-                # we want stream but not fetch, so DVC out directories are
-                # walked, but dir contents is not fetched
-                tree = RepoTree(repo, stream=True)
-
-                # We are polluting our repo cache with some dir listing here
-                if tree.isdir(path):
-                    return self.repo.cache.local.tree.get_hash(path, tree=tree)
-                return tree.get_file_hash(path)
+        with self._make_repo(locked=locked, stream=True) as repo:
+            path = PathInfo(os.path.join(repo.root_dir, self.def_path))
+            return repo.get_checksum(path, self.repo.cache.local)
 
     def status(self):
         current_checksum = self._get_checksum(locked=True)
@@ -88,8 +74,7 @@ def download(self, to):
                 self.def_repo[self.PARAM_REV_LOCK] = repo.get_rev()
 
             cache = self.repo.cache.local
-            with repo.use_cache(cache):
-                _, _, cache_infos = repo.fetch_external([self.def_path])
+            _, _, cache_infos = repo.fetch_external([self.def_path], cache)
             cache.checkout(to.path_info, cache_infos[0])
 
     def update(self, rev=None):