Decoupling dvcignore and fs

dvc/checkout.py

@@ -9,12 +9,14 @@
     ConfirmRemoveError,
     DvcException,
 )
+from dvc.ignore import DvcIgnoreFilter
 from dvc.objects import check, load
 from dvc.objects.errors import ObjectFormatError
 from dvc.objects.stage import stage
 from dvc.remote.slow_link_detection import (  # type: ignore[attr-defined]
     slow_link_guard,
 )
+from dvc.types import Optional
 
 logger = logging.getLogger(__name__)
 
@@ -175,8 +177,13 @@ def _checkout_file(
     return modified
 
 
-def _remove_redundant_files(path_info, fs, obj, cache, force):
-    existing_files = set(fs.walk_files(path_info))
+def _remove_redundant_files(
+    path_info, fs, obj, cache, force, dvcignore: Optional[DvcIgnoreFilter],
+):
+    if dvcignore:
+        existing_files = set(dvcignore.walk_files(fs, path_info))
+    else:
+        existing_files = set(fs.walk_files(path_info))
 
     needed_files = {path_info.joinpath(*key) for key, _ in obj}
     redundant_files = existing_files - needed_files
@@ -187,9 +194,16 @@ def _remove_redundant_files(path_info, fs, obj, cache, force):
 
 
 def _checkout_dir(
-    path_info, fs, obj, cache, force, progress_callback=None, relink=False,
+    path_info,
+    fs,
+    obj,
+    cache,
+    force,
+    progress_callback=None,
+    relink=False,
+    dvcignore: Optional[DvcIgnoreFilter] = None,
 ):
-    modified = False, False
+    modified = False
     # Create dir separately so that dir is created
     # even if there are no files in it
     if not fs.exists(path_info):
@@ -212,7 +226,10 @@ def _checkout_dir(
             modified = True
 
     modified = (
-        _remove_redundant_files(path_info, fs, obj, cache, force) or modified
+        _remove_redundant_files(
+            path_info, fs, obj, cache, force, dvcignore=dvcignore
+        )
+        or modified
     )
 
     fs.repo.state.save(path_info, fs, obj.hash_info)
@@ -229,14 +246,22 @@ def _checkout(
     force=False,
     progress_callback=None,
     relink=False,
+    dvcignore: Optional[DvcIgnoreFilter] = None,
 ):
     if not obj.hash_info.isdir:
         ret = _checkout_file(
             path_info, fs, obj, cache, force, progress_callback, relink
         )
     else:
         ret = _checkout_dir(
-            path_info, fs, obj, cache, force, progress_callback, relink,
+            path_info,
+            fs,
+            obj,
+            cache,
+            force,
+            progress_callback,
+            relink,
+            dvcignore=dvcignore,
         )
 
     fs.repo.state.save_link(path_info, fs)
@@ -253,6 +278,7 @@ def checkout(
     progress_callback=None,
     relink=False,
     quiet=False,
+    dvcignore: Optional[DvcIgnoreFilter] = None,
 ):
     if path_info.scheme not in ["local", cache.fs.scheme]:
         raise NotImplementedError
@@ -269,7 +295,7 @@ def checkout(
         failed = path_info
 
     elif not relink and not _changed(path_info, fs, obj, cache):
-        logger.trace("Data '%s' didn't change.", path_info)
+        logger.trace("Data '%s' didn't change.", path_info)  # type: ignore
         skip = True
     else:
         try:
@@ -296,5 +322,12 @@ def checkout(
     logger.debug("Checking out '%s' with cache '%s'.", path_info, obj)
 
     return _checkout(
-        path_info, fs, obj, cache, force, progress_callback, relink,
+        path_info,
+        fs,
+        obj,
+        cache,
+        force,
+        progress_callback,
+        relink,
+        dvcignore=dvcignore,
     )

dvc/command/check_ignore.py

@@ -10,7 +10,7 @@
 class CmdCheckIgnore(CmdBase):
     def __init__(self, args):
         super().__init__(args)
-        self.ignore_filter = self.repo.fs.dvcignore
+        self.ignore_filter = self.repo.dvcignore
 
     def _show_results(self, result):
         if not result.match and not self.args.non_matching:

dvc/config.py

@@ -154,7 +154,7 @@ def _load_config(self, level):
         filename = self.files[level]
         fs = self._get_fs(level)
 
-        if fs.exists(filename, use_dvcignore=False):
+        if fs.exists(filename):
             with fs.open(filename) as fobj:
                 conf_obj = ConfigObj(fobj)
         else:

dvc/dependency/repo.py

@@ -56,7 +56,6 @@ def _get_hash(self, locked=True):
                 path_info,
                 repo.repo_fs,
                 self.repo.odb.local.fs.PARAM_CHECKSUM,
-                follow_subrepos=False,
             ).hash_info
 
     def workspace_status(self):
@@ -95,16 +94,11 @@ def download(self, to, jobs=None):
             except (NoOutputOrStageError, NoRemoteError):
                 pass
             obj = stage(
-                odb,
-                path_info,
-                repo.repo_fs,
-                odb.fs.PARAM_CHECKSUM,
-                jobs=jobs,
-                follow_subrepos=False,
+                odb, path_info, repo.repo_fs, odb.fs.PARAM_CHECKSUM, jobs=jobs,
             )
             save(odb, obj, jobs=jobs)
 
-        checkout(to.path_info, to.fs, obj, odb)
+        checkout(to.path_info, to.fs, obj, odb, dvcignore=None)
 
     def update(self, rev=None):
         if rev:

dvc/dvcfile.py

@@ -124,7 +124,8 @@ def relpath(self):
         return relpath(self.path)
 
     def exists(self):
-        return self.repo.fs.exists(self.path)
+        is_ignored = self.repo.dvcignore.is_ignored_file(self.path)
+        return self.repo.fs.exists(self.path) and not is_ignored
 
     def _is_git_ignored(self):
         return is_git_ignored(self.repo, self.path)
@@ -144,8 +145,10 @@ def _load(self):
         # 3. path doesn't represent a regular file
         # 4. when the file is git ignored
         if not self.exists():
-            is_ignored = self.repo.fs.exists(self.path, use_dvcignore=False)
-            raise StageFileDoesNotExistError(self.path, dvc_ignored=is_ignored)
+            dvc_ignored = self.repo.dvcignore.is_ignored_file(self.path)
+            raise StageFileDoesNotExistError(
+                self.path, dvc_ignored=dvc_ignored
+            )
 
         self._verify_filename()
         if not self.repo.fs.isfile(self.path):

dvc/fs/azure.py

@@ -53,7 +53,7 @@ class AzureAuthError(DvcException):
     pass
 
 
-class AzureFileSystem(FSSpecWrapper):
+class AzureFileSystem(FSSpecWrapper):  # pylint:disable=abstract-method
     scheme = Schemes.AZURE
     PATH_CLS = CloudURLInfo
     PARAM_CHECKSUM = "etag"

dvc/fs/base.py

@@ -9,6 +9,7 @@
 from dvc.exceptions import DvcException
 from dvc.path_info import URLInfo
 from dvc.progress import Tqdm
+from dvc.scheme import Schemes
 from dvc.utils import tmp_fname
 from dvc.utils.fs import makedirs, move
 from dvc.utils.http import open_url
@@ -92,7 +93,6 @@ def get_missing_deps(cls):
         return missing
 
     def _check_requires(self):
-        from ..scheme import Schemes
         from ..utils import format_link
         from ..utils.pkg import PKG
 
@@ -139,7 +139,7 @@ def open(self, path_info, mode: str = "r", encoding: str = None, **kwargs):
 
         raise RemoteActionNotImplemented("open", self.scheme)
 
-    def exists(self, path_info, use_dvcignore=True) -> bool:
+    def exists(self, path_info) -> bool:
         raise NotImplementedError
 
     # pylint: disable=unused-argument
@@ -166,6 +166,11 @@ def iscopy(self, path_info):
         """Check if this file is an independent copy."""
         return False  # We can't be sure by default
 
+    def walk(self, top, topdown=True, onerror=None, **kwargs):
+        """Return a generator with (root, dirs, files).
+        """
+        raise NotImplementedError
+
     def walk_files(self, path_info, **kwargs):
         """Return a generator with `PathInfo`s to all the files.
 

dvc/fs/fsspec_wrapper.py

@@ -2,6 +2,8 @@
 import shutil
 from functools import lru_cache
 
+from funcy import cached_property
+
 from dvc.progress import Tqdm
 
 from .base import BaseFileSystem
@@ -14,6 +16,7 @@ def __init__(self, repo, config):
         self.fs_args = {"skip_instance_cache": True}
         self.fs_args.update(self._prepare_credentials(config))
 
+    @cached_property
     def fs(self):
         raise NotImplementedError
 
@@ -88,7 +91,7 @@ def open(
     def copy(self, from_info, to_info):
         self.fs.copy(self._with_bucket(from_info), self._with_bucket(to_info))
 
-    def exists(self, path_info, use_dvcignore=False):
+    def exists(self, path_info) -> bool:
         return self.fs.exists(self._with_bucket(path_info))
 
     def ls(self, path_info, detail=False):

dvc/fs/gdrive.py

@@ -85,7 +85,7 @@ def __init__(self, url):
         self._spath = re.sub("/{2,}", "/", self._spath.rstrip("/"))
 
 
-class GDriveFileSystem(BaseFileSystem):
+class GDriveFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
     scheme = Schemes.GDRIVE
     PATH_CLS = GDriveURLInfo
     PARAM_CHECKSUM = "checksum"
@@ -517,7 +517,7 @@ def _get_item_id(self, path_info, create=False, use_cache=True, hint=None):
         assert not create
         raise FileMissingError(path_info, hint)
 
-    def exists(self, path_info, use_dvcignore=True):
+    def exists(self, path_info) -> bool:
         try:
             self._get_item_id(path_info)
         except FileMissingError: