repofs: base on fsspec

dvc/api.py

@@ -19,21 +19,17 @@ def get_url(path, repo=None, rev=None, remote=None):
     directory in the remote storage.
     """
     with Repo.open(repo, rev=rev, subrepos=True, uninitialized=True) as _repo:
-        fs_path = _repo.fs.path.join(_repo.root_dir, path)
         with reraise(FileNotFoundError, PathMissingError(path, repo)):
-            info = _repo.repo_fs.info(fs_path)
+            info = _repo.repo_fs.info(path)
 
-        if not info["isdvc"]:
+        dvc_info = info.get("dvc_info")
+        if not dvc_info:
             raise OutputNotFoundError(path, repo)
 
-        cloud = info["repo"].cloud
-        dvc_path = _repo.fs.path.relpath(fs_path, info["repo"].root_dir)
+        dvc_repo = info["repo"]
+        md5 = dvc_info["md5"]
 
-        if not os.path.isabs(path):
-            dvc_path = dvc_path.replace("\\", "/")
-
-        md5 = info["repo"].dvcfs.info(dvc_path)["md5"]
-        return cloud.get_url_for(remote, checksum=md5)
+        return dvc_repo.cloud.get_url_for(remote, checksum=md5)
 
 
 def open(  # noqa, pylint: disable=redefined-builtin

dvc/data/reference.py

@@ -52,17 +52,12 @@ def _get_checksum(self) -> str:
         return self.fs.checksum(self.fs_path)
 
     def to_bytes(self):
-        from dvc.fs.repo import RepoFileSystem
-
         # NOTE: dumping reference FS's this way is insecure, as the
         # fully parsed remote FS config will include credentials
         #
         # ReferenceHashFiles should currently only be serialized in
         # memory and not to disk
         fs_path = self.fs_path
-        if isinstance(self.fs, RepoFileSystem):
-            fs_path = self.fs.path.relpath(fs_path, self.fs.root_dir)
-
         dict_ = {
             self.PARAM_PATH: fs_path,
             self.PARAM_HASH: self.hash_info,
@@ -77,7 +72,7 @@ def to_bytes(self):
     @classmethod
     def from_bytes(cls, data: bytes, fs_cache: Optional[dict] = None):
         from dvc.fs import get_fs_cls
-        from dvc.fs.repo import RepoFileSystem
+        from dvc.fs.repo import RepoFileSystem, _RepoFileSystem
 
         try:
             dict_ = pickle.loads(data)
@@ -94,12 +89,11 @@ def from_bytes(cls, data: bytes, fs_cache: Optional[dict] = None):
         fs = fs_cache.get((scheme, config_pairs)) if fs_cache else None
         if not fs:
             config = dict(config_pairs)
-            if RepoFileSystem.PARAM_REPO_URL in config:
-                fs = RepoFileSystem(**config)
-                fs_path = fs.path.join(fs.root_dir, fs_path)
+            if _RepoFileSystem.PARAM_REPO_URL in config:
+                fs_cls = RepoFileSystem
             else:
                 fs_cls = get_fs_cls(config, scheme=scheme)
-                fs = fs_cls(**config)
+            fs = fs_cls(**config)
         return ReferenceHashFile(
             fs_path,
             fs,

dvc/dependency/repo.py

@@ -101,6 +101,7 @@ def _get_used_and_obj(
         from dvc.data.stage import stage
         from dvc.data.tree import Tree, TreeError
         from dvc.exceptions import NoOutputOrStageError, PathMissingError
+        from dvc.utils import as_posix
 
         local_odb = self.repo.odb.local
         locked = kwargs.pop("locked", True)
@@ -112,11 +113,10 @@ def _get_used_and_obj(
             if locked and self.def_repo.get(self.PARAM_REV_LOCK) is None:
                 self.def_repo[self.PARAM_REV_LOCK] = rev
 
-            path = os.path.abspath(os.path.join(repo.root_dir, self.def_path))
             if not obj_only:
                 try:
                     for odb, obj_ids in repo.used_objs(
-                        [path],
+                        [os.path.join(repo.root_dir, self.def_path)],
                         force=True,
                         jobs=kwargs.get("jobs"),
                         recursive=True,
@@ -132,7 +132,7 @@ def _get_used_and_obj(
             try:
                 staging, _, staged_obj = stage(
                     local_odb,
-                    path,
+                    as_posix(self.def_path),
                     repo.repo_fs,
                     local_odb.fs.PARAM_CHECKSUM,
                 )
@@ -172,17 +172,17 @@ def iter_objs():
                 continue
             if (
                 obj.fs.repo_url in checked_urls
-                or obj.fs.root_dir in checked_urls
+                or obj.fs.repo.root_dir in checked_urls
             ):
                 continue
             self_url = self.repo.url or self.repo.root_dir
             if (
                 obj.fs.repo_url is not None
                 and obj.fs.repo_url == self_url
-                or obj.fs.root_dir == self.repo.root_dir
+                or obj.fs.repo.root_dir == self.repo.root_dir
             ):
                 raise CircularImportError(self, obj.fs.repo_url, self_url)
-            checked_urls.update([obj.fs.repo_url, obj.fs.root_dir])
+            checked_urls.update([obj.fs.repo_url, obj.fs.repo.root_dir])
 
     def get_obj(self, filter_info=None, **kwargs):
         locked = kwargs.get("locked", True)

dvc/fs/fsspec_wrapper.py

@@ -91,7 +91,7 @@ def ls(self, path: AnyFSPath, detail: "Literal[False]") -> Iterator[str]:
         ...
 
     def ls(self, path, detail=False):
-        yield from self.fs.ls(path, detail=detail)
+        return self.fs.ls(path, detail=detail)
 
     def find(self, path, prefix=None):
         yield from self.fs.find(path)

dvc/fs/git.py

@@ -50,3 +50,6 @@ def fs(self) -> "FsspecGitFileSystem":
     @property
     def rev(self) -> str:
         return self.fs.rev
+
+    def ls(self, path, detail=True, **kwargs):
+        return self.fs.ls(path, detail=detail, **kwargs) or []

dvc/fs/local.py

@@ -147,6 +147,9 @@ def reflink(self, from_info, to_info):
     def info(self, path):
         return self.fs.info(path)
 
+    def ls(self, path, **kwargs):
+        return self.fs.ls(path, **kwargs)
+
     def put_file(
         self, from_file, to_info, callback=DEFAULT_CALLBACK, **kwargs
     ):

dvc/fs/path.py

@@ -81,7 +81,6 @@ def overlaps(self, left, right):
         return self.isin_or_eq(left, right) or self.isin(right, left)
 
     def relpath(self, path, start):
-        assert start
         return self.flavour.relpath(path, start=start)
 
     def relparts(self, path, base):