From f5b287ee9f95c7150df6eee5f0213b11d08ba7bf Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sun, 21 Sep 2025 17:05:46 +0200 Subject: [PATCH 01/32] tests: add relative path tests --- upath/tests/test_relative.py | 212 +++++++++++++++++++++++++++++++++++ 1 file changed, 212 insertions(+) create mode 100644 upath/tests/test_relative.py diff --git a/upath/tests/test_relative.py b/upath/tests/test_relative.py new file mode 100644 index 00000000..64530c05 --- /dev/null +++ b/upath/tests/test_relative.py @@ -0,0 +1,212 @@ +"""Tests for relative path functionality.""" + +import os +import pickle +import tempfile + +import pytest + +from upath import UPath + + +@pytest.mark.parametrize( + "pth,base,rel", + [ + ("/foo/bar/baz.txt", "/foo", "bar/baz.txt"), + ("/foo/bar/baz/qux.txt", "/foo/bar", "baz/qux.txt"), + ("/foo/bar/baz/qux.txt", "/foo/bar/baz", "qux.txt"), + ("/foo/bar/baz", "/foo/bar/baz", "."), + ], +) +@pytest.mark.parametrize( + "protocol", + [ + "memory", + "file", + "", + ], +) +def test_basic_relative_path_creation(protocol, pth, base, rel): + rel_pth = UPath(pth, protocol=protocol).relative_to(UPath(base, protocol=protocol)) + + assert not rel_pth.is_absolute() + assert str(rel_pth) == rel + + +def test_relative_path_validation(): + """Test validation of relative_to arguments.""" + p = UPath("memory:///foo/bar") + + # Different protocols should fail + with pytest.raises(ValueError, match="different storage_options"): + p.relative_to(UPath("s3://bucket")) + + # Different storage options should fail + with pytest.raises(ValueError, match="different storage_options"): + UPath("s3://bucket/file", anon=True).relative_to( + UPath("s3://bucket", anon=False) + ) + + +def test_path_not_in_subpath(): + """Test relative_to with paths that don't have a parent-child relationship.""" + p = UPath("memory:///foo/bar") + other = UPath("memory:///baz") + + with pytest.raises(ValueError, match="is not in the subpath of"): + p.relative_to(other) + + +def test_filesystem_operations_fail_without_cwd(): + """Test that filesystem operations fail on relative paths when cwd()""" + p = UPath("memory:///foo/bar/baz.txt") + root = UPath("memory:///foo") + rel = p.relative_to(root) + + # Memory filesystem doesn't implement cwd(), so these should fail + with pytest.raises( + NotImplementedError, + match="require cwd\\(\\) to be implemented", + ): + _ = rel.path + + with pytest.raises( + NotImplementedError, match="require cwd\\(\\) to be implemented" + ): + rel.exists() + + +def test_filesystem_operations_work_with_cwd(): + """Test that filesystem operations work on relative paths when cwd()""" + with tempfile.TemporaryDirectory() as tmpdir: + # Create test file structure + test_dir = os.path.join(tmpdir, "testdir") + os.makedirs(test_dir, exist_ok=True) + test_file = os.path.join(test_dir, "testfile.txt") + with open(test_file, "w") as f: + f.write("test content") + + # Create paths + abs_path = UPath(test_file) + abs_dir = UPath(test_dir) + rel_path = abs_path.relative_to(abs_dir) + + assert not rel_path.is_absolute() + assert str(rel_path) == "testfile.txt" + + # Change to the test directory and try filesystem operations + old_cwd = os.getcwd() + try: + os.chdir(test_dir) + + # These should work now since we're in the right directory + full_path = rel_path.path + assert "testfile.txt" in full_path + + # Test that the file exists + assert rel_path.exists() + + finally: + os.chdir(old_cwd) + + +def test_pickling_relative_paths(): + """Test that relative paths can be pickled and unpickled.""" + p = UPath("memory:///foo/bar/baz.txt") + root = UPath("memory:///foo") + rel = p.relative_to(root) + + # Pickle and unpickle + pickled = pickle.dumps(rel) + unpickled = pickle.loads(pickled) + + assert str(rel) == str(unpickled) + assert rel.is_absolute() == unpickled.is_absolute() + assert rel._relative_base == unpickled._relative_base + + +def test_with_segments_preserves_relative_state(): + """Test that with_segments preserves the relative state.""" + p = UPath("memory:///foo/bar/baz.txt") + root = UPath("memory:///foo") + rel = p.relative_to(root) + + # Create new path with different segments + new_rel = rel.with_segments("memory:///foo/other/file.txt") + + # Should still be relative with same root + assert not new_rel.is_absolute() + assert new_rel._relative_base == rel._relative_base + + +def test_relative_path_parts(): + """Test that parts work correctly for relative paths.""" + p = UPath("memory:///foo/bar/baz/qux.txt") + root = UPath("memory:///foo") + rel = p.relative_to(root) + + assert p.parts == root.parts + rel.parts + + +def test_absolute_method_behavior(): + """Test that absolute() returns the original absolute path.""" + p = UPath("memory:///foo/bar/baz.txt") + root = UPath("memory:///foo") + rel = p.relative_to(root) + + with pytest.raises( + NotImplementedError, + match="require cwd\\(\\) to be implemented", + ): + rel.absolute() + + +def test_is_absolute_method(): + """Test is_absolute() method on relative paths.""" + p = UPath("memory:///foo/bar/baz.txt") + root = UPath("memory:///foo") + rel = p.relative_to(root) + + assert not rel.is_absolute() + + +def test_relative_path_comparison(): + """Test that relative paths can be compared.""" + p1 = UPath("memory:///foo/bar/baz.txt") + p2 = UPath("memory:///foo/bar/qux.txt") + root = UPath("memory:///foo") + + rel1 = p1.relative_to(root) + rel2 = p2.relative_to(root) + + # Compare string representations since .path requires cwd() for memory:// + assert str(rel1) != str(rel2) + assert rel1 != rel2 + + # Same relative path should be equal + rel1_copy = p1.relative_to(root) + assert str(rel1) == str(rel1_copy) + assert rel1 == rel1_copy + + # Same relative path from different base should be equal + rel3 = UPath("memory:///a/b/c.txt").relative_to(UPath("memory:///a")) + rel4 = UPath("file:///x/b/c.txt").relative_to(UPath("file:///x")) + + assert str(rel3) == str(rel4) + assert rel3 == rel4 + + +def test_nonrelative_path_is_absolute(): + """Test that normal (non-relative) paths return True for is_absolute().""" + p = UPath("memory:///foo/bar/baz.txt") + assert p.is_absolute() + + +def test_s3_relative_paths(): + """Test relative paths work with S3 URLs.""" + p = UPath("s3://test_bucket/dir/file.txt") + root = UPath("s3://test_bucket") + rel = p.relative_to(root) + + assert not rel.is_absolute() + assert str(rel) == "dir/file.txt" From 04b2ab8cab8f9b62d3be5bfb0f718e65e0d9a885 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sun, 21 Sep 2025 17:07:36 +0200 Subject: [PATCH 02/32] upath: initial relative path implementation --- upath/core.py | 126 +++++++++++++++++++++++++++++++-- upath/implementations/cloud.py | 5 -- 2 files changed, 121 insertions(+), 10 deletions(-) diff --git a/upath/core.py b/upath/core.py index 3ab82c38..1d111b54 100644 --- a/upath/core.py +++ b/upath/core.py @@ -71,7 +71,12 @@ def _check_fsspec_has_working_glob(): def _make_instance(cls, args, kwargs): """helper for pickling UPath instances""" - return cls(*args, **kwargs) + # Extract _relative_base if present + relative_base = kwargs.pop("_relative_base", None) + instance = cls(*args, **kwargs) + if relative_base is not None: + instance._relative_base = relative_base + return instance def _buffering2blocksize(mode: str, buffering: int) -> int | None: @@ -196,7 +201,24 @@ def fs(self) -> AbstractFileSystem: @property def path(self) -> str: """The path that a fsspec filesystem can use.""" - return self.parser.strip_protocol(self.__str__()) + if self._relative_base is not None: + # For relative paths, we need to resolve to absolute path + try: + current_dir = self.cwd() + except NotImplementedError: + raise NotImplementedError( + f"Filesystem operations on relative {self.__class__.__name__} " + "require cwd() to be implemented" + ) + else: + # Join the current directory with the relative path + if str(self) == ".": + path = current_dir + else: + path = current_dir / str(self) + else: + path = self.__str__() + return self.parser.strip_protocol(path) def joinuri(self, uri: JoinablePathLike) -> UPath: """Join with urljoin behavior for UPath instances""" @@ -378,6 +400,7 @@ def __init__( self._chain = Chain.from_list(segments) self._chain_parser = chain_parser self._raw_urlpaths = args + self._relative_base = None # --- deprecated attributes --------------------------------------- @@ -395,6 +418,7 @@ class UPath(_UPathMixin, OpenablePath): "_chain_parser", "_fs_cached", "_raw_urlpaths", + "_relative_base", ) if TYPE_CHECKING: @@ -402,22 +426,44 @@ class UPath(_UPathMixin, OpenablePath): _chain_parser: FSSpecChainParser _fs_cached: bool _raw_urlpaths: Sequence[JoinablePathLike] + _relative_base: str | None # === JoinablePath attributes ===================================== parser: UPathParser = LazyFlavourDescriptor() # type: ignore[assignment] def with_segments(self, *pathsegments: JoinablePathLike) -> Self: - return type(self)( + new_instance = type(self)( *pathsegments, protocol=self._protocol, **self._storage_options, ) + # Preserve _relative_base if it was set + if hasattr(self, "_relative_base") and self._relative_base is not None: + new_instance._relative_base = self._relative_base + return new_instance def __str__(self) -> str: return self.__vfspath__() def __vfspath__(self) -> str: + if self._relative_base is not None: + full_path = self._chain_parser.chain(self._chain.to_list())[0] + root_path = self._relative_base + + # Strip protocol for comparison + full_path_no_protocol = self.parser.strip_protocol(full_path) + root_path_no_protocol = self.parser.strip_protocol(root_path) + + # Calculate relative path from root to this path + if full_path_no_protocol.startswith(root_path_no_protocol): + rel_path = full_path_no_protocol[len(root_path_no_protocol) :].lstrip( + self.parser.sep + ) + return rel_path or "." + else: + # If paths don't have the expected relationship, fall back + return full_path return self._chain_parser.chain(self._chain.to_list())[0] def __repr__(self) -> str: @@ -427,6 +473,13 @@ def __repr__(self) -> str: @property def parts(self) -> Sequence[str]: + # For relative paths, return parts of the relative path only + if self._relative_base is not None: + rel_str = str(self) + if rel_str == ".": + return () + return tuple(rel_str.split(self.parser.sep)) + split = self.parser.split sep = self.parser.sep @@ -745,16 +798,43 @@ def group(self) -> str: raise NotImplementedError def absolute(self) -> Self: + if self._relative_base is not None: + try: + return self.cwd().joinpath(str(self)) + except NotImplementedError: + raise NotImplementedError( + f"Filesystem operations on relative {self.__class__.__name__} " + "require cwd() to be implemented" + ) return self def is_absolute(self) -> bool: - return self.parser.isabs(str(self)) + if self._relative_base is not None: + return False + else: + return self.parser.isabs(str(self)) def __eq__(self, other: object) -> bool: """UPaths are considered equal if their protocol, path and storage_options are equal.""" if not isinstance(other, UPath): return NotImplemented + + # For relative paths, compare the string representation instead of path + if ( + self._relative_base is not None + or getattr(other, "_relative_base", None) is not None + ): + # If both are relative paths, compare just the relative strings + if ( + self._relative_base is not None + and getattr(other, "_relative_base", None) is not None + ): + return str(self) == str(other) + else: + # One is relative, one is not - they can't be equal + return False + return ( self.path == other.path and self.protocol == other.protocol @@ -895,6 +975,9 @@ def __reduce__(self): "protocol": self._protocol, **self._storage_options, } + # Include _relative_base in the state if it's set + if self._relative_base is not None: + kwargs["_relative_base"] = self._relative_base return _make_instance, (type(self), args, kwargs) def as_uri(self) -> str: @@ -940,7 +1023,40 @@ def relative_to( # type: ignore[override] "paths have different storage_options:" f" {self.storage_options!r} != {other.storage_options!r}" ) - return self # super().relative_to(other, *_deprecated, walk_up=walk_up) + + # Calculate the relative path properly + other_str = str(other) + self_str = str(self) + + # Normalize paths by ensuring root path ends with separator if it should + # Check if self starts with other as a proper path component + if self_str == other_str: + # Same path - return "." + new_instance = copy(self) + new_instance._relative_base = other_str + return new_instance + + # Check if self_str starts with other_str followed by a separator + sep = self.parser.sep + if self_str.startswith(other_str + sep): + # Valid subpath + new_instance = copy(self) + new_instance._relative_base = other_str + return new_instance + elif self_str.startswith(other_str) and len(self_str) > len(other_str): + # Check if the next character is a separator + # (for cases where other_str ends with sep) + next_char = self_str[len(other_str)] + if next_char == sep: + new_instance = copy(self) + new_instance._relative_base = other_str + return new_instance + + # Not a valid subpath + if not walk_up: + raise ValueError(f"{self_str!r} is not in the subpath of {other_str!r}") + # For walk_up=True, we'd need more complex logic - for now, keep simple + raise ValueError(f"{self_str!r} is not in the subpath of {other_str!r}") def is_relative_to(self, other, /, *_deprecated) -> bool: # type: ignore[override] if isinstance(other, UPath) and self.storage_options != other.storage_options: diff --git a/upath/implementations/cloud.py b/upath/implementations/cloud.py index fabb7383..ca0749b2 100644 --- a/upath/implementations/cloud.py +++ b/upath/implementations/cloud.py @@ -60,11 +60,6 @@ def iterdir(self) -> Iterator[Self]: raise NotADirectoryError(str(self)) yield from super().iterdir() - def relative_to(self, other, /, *_deprecated, walk_up=False): - # use the parent implementation for the ValueError logic - super().relative_to(other, *_deprecated, walk_up=False) - return self - class GCSPath(CloudPath): __slots__ = () From 8f95ea61719e78ee585f8940355fe87b7428bfb5 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sun, 21 Sep 2025 17:08:19 +0200 Subject: [PATCH 03/32] tests: adjust relative_to tests --- upath/tests/implementations/test_s3.py | 2 +- upath/tests/test_core.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/upath/tests/implementations/test_s3.py b/upath/tests/implementations/test_s3.py index c18f089a..2029e1bd 100644 --- a/upath/tests/implementations/test_s3.py +++ b/upath/tests/implementations/test_s3.py @@ -52,7 +52,7 @@ def test_rmdir(self): self.path.joinpath("file1.txt").rmdir() def test_relative_to(self): - assert "s3://test_bucket/file.txt" == str( + assert "file.txt" == str( UPath("s3://test_bucket/file.txt").relative_to(UPath("s3://test_bucket")) ) diff --git a/upath/tests/test_core.py b/upath/tests/test_core.py index 02183480..57be7f13 100644 --- a/upath/tests/test_core.py +++ b/upath/tests/test_core.py @@ -340,7 +340,7 @@ def test_copy_path_append_kwargs(): def test_relative_to(): - assert "s3://test_bucket/file.txt" == str( + assert "file.txt" == str( UPath("s3://test_bucket/file.txt").relative_to(UPath("s3://test_bucket")) ) From 067bdf7f70334322960f665054ef322b79fbfeb3 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sun, 21 Sep 2025 18:00:35 +0200 Subject: [PATCH 04/32] Various fixes for windows and typesafety --- upath/core.py | 78 +++++++++++++++++++++------------- upath/implementations/local.py | 6 +++ upath/tests/test_relative.py | 6 +-- 3 files changed, 58 insertions(+), 32 deletions(-) diff --git a/upath/core.py b/upath/core.py index 1d111b54..e93e8817 100644 --- a/upath/core.py +++ b/upath/core.py @@ -175,6 +175,21 @@ def _raw_urlpaths(self) -> Sequence[JoinablePathLike]: def _raw_urlpaths(self, value: Sequence[JoinablePathLike]) -> None: raise NotImplementedError + @property + @abstractmethod + def _relative_base(self) -> str | None: + raise NotImplementedError + + @_relative_base.setter + def _relative_base(self, value: str | None) -> None: + raise NotImplementedError + + @classmethod + @abstractmethod + def cwd(cls) -> Self: + """Return a new path representing the current working directory.""" + raise NotImplementedError + # === upath.UPath PUBLIC ADDITIONAL API =========================== @property @@ -212,12 +227,12 @@ def path(self) -> str: ) else: # Join the current directory with the relative path - if str(self) == ".": - path = current_dir + if (self_path := str(self)) == ".": + path = str(current_dir) else: - path = current_dir / str(self) + path = current_dir.parser.join(str(self), self_path) else: - path = self.__str__() + path = str(self) return self.parser.strip_protocol(path) def joinuri(self, uri: JoinablePathLike) -> UPath: @@ -995,34 +1010,41 @@ def samefile(self, other_path) -> bool: return st == other_st @classmethod - def cwd(cls) -> UPath: + def cwd(cls) -> Self: if cls is UPath: - return get_upath_class("").cwd() # type: ignore[union-attr] + # default behavior for UPath.cwd() is to return local cwd + return get_upath_class("").cwd() # type: ignore[union-attr,return-value] else: raise NotImplementedError @classmethod - def home(cls) -> UPath: + def home(cls) -> Self: if cls is UPath: - return get_upath_class("").home() # type: ignore[union-attr] + return get_upath_class("").home() # type: ignore[union-attr,return-value] else: raise NotImplementedError def relative_to( # type: ignore[override] self, - other, + other: Self | str, /, *_deprecated, - walk_up=False, + walk_up: bool = False, ) -> Self: - if isinstance(other, UPath) and ( - (self.__class__ is not other.__class__) - or (self.storage_options != other.storage_options) - ): - raise ValueError( - "paths have different storage_options:" - f" {self.storage_options!r} != {other.storage_options!r}" - ) + if walk_up: + raise NotImplementedError("walk_up=True is not implemented yet") + + if isinstance(other, UPath): + if self.__class__ is not other.__class__: + raise ValueError( + "incompatible protocols:" + f" {self._protocol!r} != {other._protocol!r}" + ) + if self.storage_options != other.storage_options: + raise ValueError( + "incompatible storage_options:" + f" {self.storage_options!r} != {other.storage_options!r}" + ) # Calculate the relative path properly other_str = str(other) @@ -1043,19 +1065,17 @@ def relative_to( # type: ignore[override] new_instance = copy(self) new_instance._relative_base = other_str return new_instance - elif self_str.startswith(other_str) and len(self_str) > len(other_str): + elif ( + self_str.startswith(other_str) + and len(self_str) > len(other_str) + and self_str[len(other_str)] == sep + ): # Check if the next character is a separator # (for cases where other_str ends with sep) - next_char = self_str[len(other_str)] - if next_char == sep: - new_instance = copy(self) - new_instance._relative_base = other_str - return new_instance - - # Not a valid subpath - if not walk_up: - raise ValueError(f"{self_str!r} is not in the subpath of {other_str!r}") - # For walk_up=True, we'd need more complex logic - for now, keep simple + new_instance = copy(self) + new_instance._relative_base = other_str + return new_instance + raise ValueError(f"{self_str!r} is not in the subpath of {other_str!r}") def is_relative_to(self, other, /, *_deprecated) -> bool: # type: ignore[override] diff --git a/upath/implementations/local.py b/upath/implementations/local.py index 8d71c7be..26aaa360 100644 --- a/upath/implementations/local.py +++ b/upath/implementations/local.py @@ -71,11 +71,13 @@ class LocalPath(_UPathMixin, pathlib.Path): "_chain", "_chain_parser", "_fs_cached", + "_relative_base", ) if TYPE_CHECKING: _chain: Chain _chain_parser: FSSpecChainParser _fs_cached: AbstractFileSystem + _relative_base: str | None parser = os.path # type: ignore[misc,assignment] @@ -175,6 +177,10 @@ def __rtruediv__(self, other) -> Self: raise ValueError("can't combine incompatible UPath protocols") return super().__rtruediv__(other) + @classmethod + def cwd(cls) -> Self: + return cls(super().cwd()) + UPath.register(LocalPath) diff --git a/upath/tests/test_relative.py b/upath/tests/test_relative.py index 64530c05..c9f5578a 100644 --- a/upath/tests/test_relative.py +++ b/upath/tests/test_relative.py @@ -30,7 +30,7 @@ def test_basic_relative_path_creation(protocol, pth, base, rel): rel_pth = UPath(pth, protocol=protocol).relative_to(UPath(base, protocol=protocol)) assert not rel_pth.is_absolute() - assert str(rel_pth) == rel + assert rel_pth.as_posix() == rel def test_relative_path_validation(): @@ -38,11 +38,11 @@ def test_relative_path_validation(): p = UPath("memory:///foo/bar") # Different protocols should fail - with pytest.raises(ValueError, match="different storage_options"): + with pytest.raises(ValueError, match="incompatible protocols"): p.relative_to(UPath("s3://bucket")) # Different storage options should fail - with pytest.raises(ValueError, match="different storage_options"): + with pytest.raises(ValueError, match="incompatible storage_options"): UPath("s3://bucket/file", anon=True).relative_to( UPath("s3://bucket", anon=False) ) From 66b8d2b7ec92005f518c8ac06f0d4adf9ce8ebb0 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Fri, 26 Sep 2025 20:51:02 +0200 Subject: [PATCH 05/32] upath: don't make cwd() abstract in mixin --- upath/core.py | 8 +------- upath/implementations/local.py | 4 ---- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/upath/core.py b/upath/core.py index e93e8817..b1f91d09 100644 --- a/upath/core.py +++ b/upath/core.py @@ -184,12 +184,6 @@ def _relative_base(self) -> str | None: def _relative_base(self, value: str | None) -> None: raise NotImplementedError - @classmethod - @abstractmethod - def cwd(cls) -> Self: - """Return a new path representing the current working directory.""" - raise NotImplementedError - # === upath.UPath PUBLIC ADDITIONAL API =========================== @property @@ -219,7 +213,7 @@ def path(self) -> str: if self._relative_base is not None: # For relative paths, we need to resolve to absolute path try: - current_dir = self.cwd() + current_dir = self.cwd() # type: ignore[attr-defined] except NotImplementedError: raise NotImplementedError( f"Filesystem operations on relative {self.__class__.__name__} " diff --git a/upath/implementations/local.py b/upath/implementations/local.py index 26aaa360..a2b07af7 100644 --- a/upath/implementations/local.py +++ b/upath/implementations/local.py @@ -177,10 +177,6 @@ def __rtruediv__(self, other) -> Self: raise ValueError("can't combine incompatible UPath protocols") return super().__rtruediv__(other) - @classmethod - def cwd(cls) -> Self: - return cls(super().cwd()) - UPath.register(LocalPath) From 07bd61510912e617e1c7285d01ca0e3d73c20a6a Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Fri, 26 Sep 2025 22:31:31 +0200 Subject: [PATCH 06/32] upath: as_uri() for relative paths --- upath/core.py | 6 ++++++ upath/tests/test_relative.py | 15 +++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/upath/core.py b/upath/core.py index b1f91d09..2be55f18 100644 --- a/upath/core.py +++ b/upath/core.py @@ -525,6 +525,8 @@ def with_name(self, name) -> Self: @property def anchor(self) -> str: + if self._relative_base is not None: + return "" return self.drive + self.root # === ReadablePath attributes ===================================== @@ -990,6 +992,10 @@ def __reduce__(self): return _make_instance, (type(self), args, kwargs) def as_uri(self) -> str: + if self._relative_base is not None: + raise ValueError( + f"relative path can't be expressed as a {self.protocol} URI" + ) return str(self) def as_posix(self) -> str: diff --git a/upath/tests/test_relative.py b/upath/tests/test_relative.py index c9f5578a..e702e424 100644 --- a/upath/tests/test_relative.py +++ b/upath/tests/test_relative.py @@ -210,3 +210,18 @@ def test_s3_relative_paths(): assert not rel.is_absolute() assert str(rel) == "dir/file.txt" + + +@pytest.fixture +def rel_path(): + p = UPath("memory:///foo/bar/baz.txt") + root = UPath("memory:///foo") + yield p.relative_to(root) + + +def test_relative_path_as_uri(rel_path): + with pytest.raises( + ValueError, + "relative path can't be expressed as a {rel_path.protocol} URI", + ): + rel_path.as_uri() From 4eb200b26aa4fd773cc180a895db55099ede276e Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Fri, 26 Sep 2025 23:46:59 +0200 Subject: [PATCH 07/32] upath: add tests for file access on relative paths --- upath/core.py | 57 +++++++++++----------- upath/tests/test_relative.py | 93 ++++++++++++++++++++++++++++++++++-- 2 files changed, 117 insertions(+), 33 deletions(-) diff --git a/upath/core.py b/upath/core.py index 2be55f18..0e5f6813 100644 --- a/upath/core.py +++ b/upath/core.py @@ -15,6 +15,7 @@ from typing import Any from typing import BinaryIO from typing import Literal +from typing import NoReturn from typing import TextIO from typing import overload from urllib.parse import SplitResult @@ -92,6 +93,11 @@ def _buffering2blocksize(mode: str, buffering: int) -> int | None: return buffering +def _raise_unsupported(cls_name: str, method: str) -> NoReturn: + "relative path does not support method(), because cls_name.cwd() is unsupported" + raise NotImplementedError(f"{cls_name}.{method}() is unsupported") + + class _UPathMeta(ABCMeta): if sys.version_info < (3, 11): # pathlib 3.9 and 3.10 supported `Path[str]` but @@ -212,19 +218,12 @@ def path(self) -> str: """The path that a fsspec filesystem can use.""" if self._relative_base is not None: # For relative paths, we need to resolve to absolute path - try: - current_dir = self.cwd() # type: ignore[attr-defined] - except NotImplementedError: - raise NotImplementedError( - f"Filesystem operations on relative {self.__class__.__name__} " - "require cwd() to be implemented" - ) + current_dir = self.cwd() # type: ignore[attr-defined] + # Join the current directory with the relative path + if (self_path := str(self)) == ".": + path = str(current_dir) else: - # Join the current directory with the relative path - if (self_path := str(self)) == ".": - path = str(current_dir) - else: - path = current_dir.parser.join(str(self), self_path) + path = current_dir.parser.join(str(self), self_path) else: path = str(self) return self.parser.strip_protocol(path) @@ -533,7 +532,7 @@ def anchor(self) -> str: @property def info(self) -> PathInfo: - raise NotImplementedError("todo") + _raise_unsupported(type(self).__name__, "info") def iterdir(self) -> Iterator[Self]: sep = self.parser.sep @@ -555,7 +554,7 @@ def __open_reader__(self) -> BinaryIO: return self.fs.open(self.path, mode="rb") def readlink(self) -> Self: - raise NotImplementedError + _raise_unsupported(type(self).__name__, "readlink") # --- WritablePath attributes ------------------------------------- @@ -564,7 +563,7 @@ def symlink_to( target: ReadablePathLike, target_is_directory: bool = False, ) -> None: - raise NotImplementedError + _raise_unsupported(type(self).__name__, "symlink_to") def mkdir( self, @@ -687,7 +686,7 @@ def lstat(self) -> UPathStatResult: return self.stat(follow_symlinks=False) def chmod(self, mode: int, *, follow_symlinks: bool = True) -> None: - raise NotImplementedError + _raise_unsupported(type(self).__name__, "chmod") def exists(self, *, follow_symlinks=True) -> bool: return self.fs.exists(self.path) @@ -750,6 +749,8 @@ def glob( UserWarning, stacklevel=2, ) + if self._relative_base is not None: + self = self.absolute() path_pattern = self.joinpath(pattern).path sep = self.parser.sep base = self.fs._strip_protocol(self.path) @@ -803,20 +804,14 @@ def rglob( yield self.joinpath(name) def owner(self) -> str: - raise NotImplementedError + _raise_unsupported(type(self).__name__, "owner") def group(self) -> str: - raise NotImplementedError + _raise_unsupported(type(self).__name__, "group") def absolute(self) -> Self: if self._relative_base is not None: - try: - return self.cwd().joinpath(str(self)) - except NotImplementedError: - raise NotImplementedError( - f"Filesystem operations on relative {self.__class__.__name__} " - "require cwd() to be implemented" - ) + return self.cwd().joinpath(str(self)) return self def is_absolute(self) -> bool: @@ -881,6 +876,8 @@ def __ge__(self, other: object) -> bool: return self.path >= other.path def resolve(self, strict: bool = False) -> Self: + if self._relative_base is not None: + self = self.absolute() _parts = self.parts # Do not attempt to normalize path if no parts are dots @@ -911,7 +908,7 @@ def touch(self, mode=0o666, exist_ok=True) -> None: pass # unsupported by filesystem def lchmod(self, mode: int) -> None: - raise NotImplementedError + _raise_unsupported(type(self).__name__, "lchmod") def unlink(self, missing_ok: bool = False) -> None: if not self.exists(): @@ -939,6 +936,8 @@ def rename( target = UPath(target, **self.storage_options) if target == self: return self + if self._relative_base is not None: + self = self.absolute() target_protocol = get_upath_protocol(target) if target_protocol: if target_protocol != self.protocol: @@ -970,7 +969,7 @@ def rename( return self.with_segments(target_) def replace(self, target: WritablePathLike) -> Self: - raise NotImplementedError # todo + _raise_unsupported(type(self).__name__, "replace") @property def drive(self) -> str: @@ -1015,14 +1014,14 @@ def cwd(cls) -> Self: # default behavior for UPath.cwd() is to return local cwd return get_upath_class("").cwd() # type: ignore[union-attr,return-value] else: - raise NotImplementedError + _raise_unsupported(cls.__name__, "cwd") @classmethod def home(cls) -> Self: if cls is UPath: return get_upath_class("").home() # type: ignore[union-attr,return-value] else: - raise NotImplementedError + _raise_unsupported(cls.__name__, "home") def relative_to( # type: ignore[override] self, diff --git a/upath/tests/test_relative.py b/upath/tests/test_relative.py index e702e424..735f864d 100644 --- a/upath/tests/test_relative.py +++ b/upath/tests/test_relative.py @@ -2,6 +2,7 @@ import os import pickle +import re import tempfile import pytest @@ -66,12 +67,13 @@ def test_filesystem_operations_fail_without_cwd(): # Memory filesystem doesn't implement cwd(), so these should fail with pytest.raises( NotImplementedError, - match="require cwd\\(\\) to be implemented", + match=re.escape("MemoryPath.cwd() is unsupported"), ): _ = rel.path with pytest.raises( - NotImplementedError, match="require cwd\\(\\) to be implemented" + NotImplementedError, + match=re.escape("MemoryPath.cwd() is unsupported"), ): rel.exists() @@ -156,7 +158,7 @@ def test_absolute_method_behavior(): with pytest.raises( NotImplementedError, - match="require cwd\\(\\) to be implemented", + match=re.escape("MemoryPath.cwd() is unsupported"), ): rel.absolute() @@ -222,6 +224,89 @@ def rel_path(): def test_relative_path_as_uri(rel_path): with pytest.raises( ValueError, - "relative path can't be expressed as a {rel_path.protocol} URI", + match=f"relative path can't be expressed as a {rel_path.protocol} URI", ): rel_path.as_uri() + + +@pytest.mark.parametrize( + "method_args", + [ + pytest.param(("absolute", ()), id="absolute"), + pytest.param(("chmod", (0o777,)), id="chmod"), + pytest.param(("cwd", ()), id="cwd"), + pytest.param(("exists", ()), id="exists"), + pytest.param(("glob", ("*.txt",)), id="glob"), + pytest.param(("group", ()), id="group"), + pytest.param(("is_dir", ()), id="is_dir"), + pytest.param(("is_file", ()), id="is_file"), + pytest.param(("is_symlink", ()), id="is_symlink"), + pytest.param(("iterdir", ()), id="iterdir"), + pytest.param(("open", ()), id="open"), + pytest.param(("owner", ()), id="owner"), + pytest.param(("read_bytes", ()), id="read_bytes"), + pytest.param(("read_text", ()), id="read_text"), + pytest.param(("readlink", ()), id="readlink"), + pytest.param(("rename", ("a/b/c",)), id="rename"), + pytest.param(("replace", ("...",)), id="replace"), + pytest.param(("rglob", ("*.txt",)), id="rglob"), + pytest.param(("rmdir", ()), id="rmdir"), + pytest.param(("samefile", ("...",)), id="samefile"), + pytest.param(("stat", ()), id="stat"), + pytest.param(("symlink_to", ("...",)), id="symlink_to"), + pytest.param(("touch", ()), id="touch"), + pytest.param(("unlink", ()), id="unlink"), + pytest.param(("write_bytes", (b"data",)), id="write_bytes"), + pytest.param(("write_text", ("data",)), id="write_text"), + ], +) +def test_path_operations_disabled_without_cwd(rel_path, method_args): + """UPaths without .cwd() implementation should not allow path operations.""" + method, args = method_args + + with pytest.raises(NotImplementedError): + # next only needs to be called for iterdir and glob/rglob + # but the other raise already in the getattr call + next(getattr(rel_path, method)(*args)) + + +# 'anchor', +# 'as_posix', +# 'as_uri', +# 'drive', +# 'expanduser', +# 'fs', +# 'home', +# 'is_absolute', +# 'is_relative_to', +# 'joinpath', +# 'joinuri', +# 'lchmod', +# 'link_to', +# 'lstat', +# 'match', +# 'mkdir', +# 'name', +# 'parent', +# 'parents', +# 'parser', +# 'parts', +# 'path', +# 'protocol', +# 'relative_to', +# 'root', +# 'stem', +# 'storage_options', +# 'suffix', +# 'suffixes', +# 'with_name', +# 'with_segments', +# 'with_stem', +# 'with_suffix', +# 'is_block_device', +# 'is_char_device', +# 'is_fifo', +# 'is_mount', +# 'is_reserved', +# 'is_socket', +# 'resolve', From 78475a214b34a872c3a9a9527763bdd9dbafbf24 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sat, 27 Sep 2025 11:39:36 +0200 Subject: [PATCH 08/32] upath: relative path drive,root,anchor fixes --- upath/core.py | 4 ++ upath/implementations/cloud.py | 2 + upath/tests/test_relative.py | 78 +++++++++++++++++++++++++++++++--- 3 files changed, 78 insertions(+), 6 deletions(-) diff --git a/upath/core.py b/upath/core.py index 0e5f6813..3b9714a8 100644 --- a/upath/core.py +++ b/upath/core.py @@ -973,10 +973,14 @@ def replace(self, target: WritablePathLike) -> Self: @property def drive(self) -> str: + if self._relative_base is not None: + return "" return self.parser.splitroot(str(self))[0] @property def root(self) -> str: + if self._relative_base is not None: + return "" return self.parser.splitroot(str(self))[1] def __reduce__(self): diff --git a/upath/implementations/cloud.py b/upath/implementations/cloud.py index ca0749b2..d551e1a8 100644 --- a/upath/implementations/cloud.py +++ b/upath/implementations/cloud.py @@ -46,6 +46,8 @@ def _transform_init_args( @property def root(self) -> str: + if self._relative_base is not None: + return "" return self.parser.sep def mkdir( diff --git a/upath/tests/test_relative.py b/upath/tests/test_relative.py index 735f864d..50cc2069 100644 --- a/upath/tests/test_relative.py +++ b/upath/tests/test_relative.py @@ -270,10 +270,78 @@ def test_path_operations_disabled_without_cwd(rel_path, method_args): next(getattr(rel_path, method)(*args)) -# 'anchor', -# 'as_posix', -# 'as_uri', -# 'drive', +@pytest.mark.parametrize( + "protocol,path,base", + [ + ("", "/foo/bar/baz.txt", "/foo"), + ("file", "/foo/bar/baz.txt", "/foo"), + ("s3", "s3://bucket/foo/bar/baz.txt", "s3://bucket/foo"), + ("gcs", "gcs://bucket/foo/bar/baz.txt", "gcs://bucket/foo"), + ("ftp", "ftp://user:pass@host/foo/bar/baz.txt", "ftp://user:pass@host/foo"), + ("http", "http://host/foo/bar/baz.txt", "http://host/foo"), + ("https", "https://host/foo/bar/baz.txt", "https://host/foo"), + ("memory", "memory:///foo/bar/baz.txt", "memory:///foo"), + ], +) +def test_drive_root_anchor_empty_for_relative_paths(protocol, path, base): + rel = UPath(path, protocol=protocol).relative_to(UPath(base, protocol=protocol)) + assert (rel.drive, rel.root, rel.anchor) == ("", "", "") + + +@pytest.mark.parametrize( + "protocol,path,base,expected_rel", + [ + ("", "/foo/bar/baz.txt", "/foo", "bar/baz.txt"), + ("file", "/foo/bar/baz.txt", "/foo", "bar/baz.txt"), + ("s3", "s3://bucket/foo/bar/baz.txt", "s3://bucket/foo", "bar/baz.txt"), + ("gcs", "gcs://bucket/foo/bar/baz.txt", "gcs://bucket/foo", "bar/baz.txt"), + ( + "ftp", + "ftp://user:pass@host/foo/bar/baz.txt", + "ftp://user:pass@host/foo", + "bar/baz.txt", + ), + ("http", "http://host/foo/bar/baz.txt", "http://host/foo", "bar/baz.txt"), + ("https", "https://host/foo/bar/baz.txt", "https://host/foo", "bar/baz.txt"), + ("memory", "memory:///foo/bar/baz.txt", "memory:///foo", "bar/baz.txt"), + ], +) +def test_relative_path_properties(protocol, path, base, expected_rel): + rel = UPath(path, protocol=protocol).relative_to(UPath(base, protocol=protocol)) + + assert not rel.is_absolute() + assert rel.as_posix() == expected_rel + assert rel.parts == tuple(expected_rel.split("/")) + + +@pytest.mark.parametrize( + "protocol,path,base,expected_parts", + [ + ("", "/foo/bar/baz.txt", "/foo", ("bar", "baz.txt")), + ("file", "/foo/bar/baz.txt", "/foo", ("bar", "baz.txt")), + ("s3", "s3://bucket/foo/bar/baz.txt", "s3://bucket/foo", ("bar", "baz.txt")), + ("gcs", "gcs://bucket/foo/bar/baz.txt", "gcs://bucket/foo", ("bar", "baz.txt")), + ( + "ftp", + "ftp://user:pass@host/foo/bar/baz.txt", + "ftp://user:pass@host/foo", + ("bar", "baz.txt"), + ), + ("http", "http://host/foo/bar/baz.txt", "http://host/foo", ("bar", "baz.txt")), + ( + "https", + "https://host/foo/bar/baz.txt", + "https://host/foo", + ("bar", "baz.txt"), + ), + ("memory", "memory:///foo/bar/baz.txt", "memory:///foo", ("bar", "baz.txt")), + ], +) +def test_relative_path_parts_property(protocol, path, base, expected_parts): + rel = UPath(path, protocol=protocol).relative_to(UPath(base, protocol=protocol)) + assert rel.parts == expected_parts + + # 'expanduser', # 'fs', # 'home', @@ -290,11 +358,9 @@ def test_path_operations_disabled_without_cwd(rel_path, method_args): # 'parent', # 'parents', # 'parser', -# 'parts', # 'path', # 'protocol', # 'relative_to', -# 'root', # 'stem', # 'storage_options', # 'suffix', From 0aff83d2a09772886c40b884a6a81500f3a0ea9b Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sat, 27 Sep 2025 12:16:33 +0200 Subject: [PATCH 09/32] upath: more relative path tests --- upath/core.py | 12 ++++++-- upath/tests/test_relative.py | 55 ++++++++++++++++++++++++------------ 2 files changed, 47 insertions(+), 20 deletions(-) diff --git a/upath/core.py b/upath/core.py index 3b9714a8..3b917c30 100644 --- a/upath/core.py +++ b/upath/core.py @@ -217,8 +217,14 @@ def fs(self) -> AbstractFileSystem: def path(self) -> str: """The path that a fsspec filesystem can use.""" if self._relative_base is not None: - # For relative paths, we need to resolve to absolute path - current_dir = self.cwd() # type: ignore[attr-defined] + try: + # For relative paths, we need to resolve to absolute path + current_dir = self.cwd() # type: ignore[attr-defined] + except NotImplementedError: + raise NotImplementedError( + f"fsspec paths can not be relative and" + f" {type(self).__name__}.cwd() is unsupported" + ) from None # Join the current directory with the relative path if (self_path := str(self)) == ".": path = str(current_dir) @@ -853,6 +859,8 @@ def __hash__(self) -> int: Note: in the future, if hash collisions become an issue, we can add `fsspec.utils.tokenize(storage_options)` """ + if self._relative_base is not None: + return hash((self.protocol, str(self))) return hash((self.protocol, self.path)) def __lt__(self, other: object) -> bool: diff --git a/upath/tests/test_relative.py b/upath/tests/test_relative.py index 50cc2069..838f307c 100644 --- a/upath/tests/test_relative.py +++ b/upath/tests/test_relative.py @@ -67,13 +67,17 @@ def test_filesystem_operations_fail_without_cwd(): # Memory filesystem doesn't implement cwd(), so these should fail with pytest.raises( NotImplementedError, - match=re.escape("MemoryPath.cwd() is unsupported"), + match=re.escape( + "fsspec paths can not be relative and MemoryPath.cwd() is unsupported" + ), ): _ = rel.path with pytest.raises( NotImplementedError, - match=re.escape("MemoryPath.cwd() is unsupported"), + match=re.escape( + "fsspec paths can not be relative and MemoryPath.cwd() is unsupported" + ), ): rel.exists() @@ -342,10 +346,39 @@ def test_relative_path_parts_property(protocol, path, base, expected_parts): assert rel.parts == expected_parts -# 'expanduser', +def test_relative_path_is_something(rel_path): + assert rel_path.is_block_device() is False + assert rel_path.is_char_device() is False + assert rel_path.is_fifo() is False + assert rel_path.is_mount() is False + assert rel_path.is_reserved() is False + assert rel_path.is_socket() is False + + +def test_relative_path_hashable(): + x = UPath("memory:///a/b/c.txt") + y = x.relative_to(UPath("memory:///a")) + assert hash(y) != hash(x) + + +def test_relative_path_expanduser_noop(rel_path): + # this should be revisited if we ever add ~ support to non-file protocols + assert rel_path == rel_path.expanduser() + + +def test_relative_path_stem_suffix_name(rel_path): + assert rel_path.name == "baz.txt" + assert rel_path.stem == "baz" + assert rel_path.suffix == ".txt" + assert rel_path.suffixes == [".txt"] + assert rel_path.with_name("other.txt").name == "other.txt" + assert rel_path.with_stem("other").name == "other.txt" + assert rel_path.with_suffix(".md").name == "baz.md" + assert rel_path.with_suffix([".tar.gz"]).suffixes == [".tar", ".gz"] + + # 'fs', # 'home', -# 'is_absolute', # 'is_relative_to', # 'joinpath', # 'joinuri', @@ -354,25 +387,11 @@ def test_relative_path_parts_property(protocol, path, base, expected_parts): # 'lstat', # 'match', # 'mkdir', -# 'name', # 'parent', # 'parents', # 'parser', # 'path', # 'protocol', -# 'relative_to', -# 'stem', # 'storage_options', -# 'suffix', -# 'suffixes', -# 'with_name', # 'with_segments', -# 'with_stem', -# 'with_suffix', -# 'is_block_device', -# 'is_char_device', -# 'is_fifo', -# 'is_mount', -# 'is_reserved', -# 'is_socket', # 'resolve', From dce202b14b3cc0de9ad98d1ff8e2205cc241625b Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sat, 27 Sep 2025 12:43:52 +0200 Subject: [PATCH 10/32] tests: fix typo in relative path suffixes test --- upath/tests/test_relative.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upath/tests/test_relative.py b/upath/tests/test_relative.py index 838f307c..77326e7b 100644 --- a/upath/tests/test_relative.py +++ b/upath/tests/test_relative.py @@ -374,7 +374,7 @@ def test_relative_path_stem_suffix_name(rel_path): assert rel_path.with_name("other.txt").name == "other.txt" assert rel_path.with_stem("other").name == "other.txt" assert rel_path.with_suffix(".md").name == "baz.md" - assert rel_path.with_suffix([".tar.gz"]).suffixes == [".tar", ".gz"] + assert rel_path.with_suffix(".tar.gz").suffixes == [".tar", ".gz"] # 'fs', From 42a1188d8806a06c477202e2ed97e6feb43ce836 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sat, 27 Sep 2025 20:14:26 +0200 Subject: [PATCH 11/32] tests: add .parent tests for relative paths --- upath/tests/test_relative.py | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/upath/tests/test_relative.py b/upath/tests/test_relative.py index 77326e7b..9ca94c08 100644 --- a/upath/tests/test_relative.py +++ b/upath/tests/test_relative.py @@ -246,6 +246,9 @@ def test_relative_path_as_uri(rel_path): pytest.param(("is_file", ()), id="is_file"), pytest.param(("is_symlink", ()), id="is_symlink"), pytest.param(("iterdir", ()), id="iterdir"), + pytest.param(("lchmod", (0o777,)), id="lchmod"), + pytest.param(("lstat", ()), id="lstat"), + pytest.param(("mkdir", ()), id="mkdir"), pytest.param(("open", ()), id="open"), pytest.param(("owner", ()), id="owner"), pytest.param(("read_bytes", ()), id="read_bytes"), @@ -377,16 +380,30 @@ def test_relative_path_stem_suffix_name(rel_path): assert rel_path.with_suffix(".tar.gz").suffixes == [".tar", ".gz"] +@pytest.mark.parametrize( + "protocol,pth,base,expected_parent", + [ + ("", "/foo/bar/baz.txt", "/foo", "bar"), + ("", "/foo", "/foo", "."), + ("file", "/foo/bar/baz.txt", "/foo", "bar"), + ("file", "/foo/bar", "/foo/bar", "."), + ("s3", "s3://bucket/foo/bar/baz.txt", "s3://bucket/foo", "bar"), + ("s3", "s3://bucket/foo/bar/", "s3://bucket/foo/bar", "."), + ("gcs", "gcs://bucket/foo/bar/baz.txt", "gcs://bucket/foo", "bar"), + ("gcs", "gcs://bucket/foo/bar/", "gcs://bucket/foo", "."), + ("memory", "memory:///foo/bar/baz.txt", "memory:///foo", "bar"), + ("memory", "memory:///foo/bar", "memory:///foo", "."), + ], +) +def test_relative_path_parent(protocol, pth, base, expected_parent): + rel = UPath(pth, protocol=protocol).relative_to(UPath(base, protocol=protocol)) + assert str(rel.parent) == expected_parent + + # 'fs', # 'home', -# 'is_relative_to', # 'joinpath', # 'joinuri', -# 'lchmod', -# 'link_to', -# 'lstat', -# 'match', -# 'mkdir', # 'parent', # 'parents', # 'parser', @@ -395,3 +412,4 @@ def test_relative_path_stem_suffix_name(rel_path): # 'storage_options', # 'with_segments', # 'resolve', +# 'match', From f2ef5920acb1d54c79d7311e4c5c4020e981badc Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sat, 27 Sep 2025 20:32:59 +0200 Subject: [PATCH 12/32] upath: repr for relative paths --- upath/core.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/upath/core.py b/upath/core.py index 3b917c30..d3102265 100644 --- a/upath/core.py +++ b/upath/core.py @@ -481,6 +481,8 @@ def __vfspath__(self) -> str: return self._chain_parser.chain(self._chain.to_list())[0] def __repr__(self) -> str: + if self._relative_base is not None: + return f"" return f"{type(self).__name__}({self.path!r}, protocol={self._protocol!r})" # === JoinablePath overrides ====================================== From cb4ce5931a1d6c73cf0b45e2db8dfeb11df16b37 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sun, 28 Sep 2025 12:30:31 +0200 Subject: [PATCH 13/32] tests: relpath tests for parent(s), protocol, storage_options, path, fs and home --- upath/tests/test_relative.py | 118 ++++++++++++++++++++++++++++++++--- 1 file changed, 111 insertions(+), 7 deletions(-) diff --git a/upath/tests/test_relative.py b/upath/tests/test_relative.py index 9ca94c08..e6ab8af1 100644 --- a/upath/tests/test_relative.py +++ b/upath/tests/test_relative.py @@ -10,6 +10,60 @@ from upath import UPath +@pytest.mark.parametrize( + "protocol,storage_options,path,base", + [ + ("memory", {}, "memory:///foo/bar/baz.txt", "memory:///foo"), + ("s3", {"anon": True}, "s3://bucket/foo/bar/baz.txt", "s3://bucket/foo"), + ("gcs", {"token": "anon"}, "gcs://bucket/foo/bar/baz.txt", "gcs://bucket/foo"), + ("http", {"something": 1}, "http://host/foo/bar/baz.txt", "http://host/foo"), + ( + "https", + {}, + "https://host/foo/bar/baz.txt", + "https://host/", + ), + ], +) +def test_protocol_storage_options_fs_preserved(protocol, storage_options, path, base): + """Test that protocol and storage_options are preserved in relative paths.""" + p = UPath(path, protocol=protocol, **storage_options) + root = UPath(base, protocol=protocol, **storage_options) + rel = p.relative_to(root) + + assert rel.protocol == protocol + assert dict(**rel.storage_options) == storage_options + assert isinstance(rel.fs, type(p.fs)) + + +@pytest.mark.parametrize( + "protocol,path,base", + [ + ("s3", "s3://bucket/foo/bar/baz.txt", "s3://bucket/foo"), + ("gcs", "gcs://bucket/foo/bar/baz.txt", "gcs://bucket/foo"), + ("ftp", "ftp://user:pass@host/foo/bar/baz.txt", "ftp://user:pass@host/foo"), + ("http", "http://host/foo/bar/baz.txt", "http://host/foo"), + ("https", "https://host/foo/bar/baz.txt", "https://host/foo"), + ("memory", "memory:///foo/bar/baz.txt", "memory:///foo"), + ], +) +def test_relative_urlpath_raises_without_cwd(protocol, path, base): + rel = UPath(path, protocol=protocol).relative_to(UPath(base, protocol=protocol)) + with pytest.raises( + NotImplementedError, + match=re.escape(f"{type(rel).__name__}.cwd() is unsupported"), + ): + rel.cwd() + with pytest.raises( + NotImplementedError, + match=re.escape( + f"fsspec paths can not be relative and" + f" {type(rel).__name__}.cwd() is unsupported" + ), + ): + _ = rel.path + + @pytest.mark.parametrize( "pth,base,rel", [ @@ -393,6 +447,8 @@ def test_relative_path_stem_suffix_name(rel_path): ("gcs", "gcs://bucket/foo/bar/", "gcs://bucket/foo", "."), ("memory", "memory:///foo/bar/baz.txt", "memory:///foo", "bar"), ("memory", "memory:///foo/bar", "memory:///foo", "."), + ("https", "https://host/foo/bar/baz.txt", "https://host/foo", "bar"), + ("https", "https://host/foo/bar/", "https://host/foo/bar", "."), ], ) def test_relative_path_parent(protocol, pth, base, expected_parent): @@ -400,16 +456,64 @@ def test_relative_path_parent(protocol, pth, base, expected_parent): assert str(rel.parent) == expected_parent -# 'fs', -# 'home', +@pytest.mark.parametrize( + "uri,base,expected_parents_parts", + [ + ("/foo/bar/baz/qux.txt", "/foo", [("bar", "baz"), ("bar",), ()]), + ("file:///foo/bar/baz/qux.txt", "file:///foo", [("bar", "baz"), ("bar",), ()]), + ("s3://bucket/foo/bar/baz/", "s3://bucket/", [("foo", "bar"), ("foo",), ()]), + ("gcs://bucket/foo/bar/baz", "gcs://bucket/", [("foo", "bar"), ("foo",), ()]), + ( + "memory:///foo/bar/baz/qux.txt", + "memory:///foo", + [("bar", "baz"), ("bar",), ()], + ), + ( + "https://host.com/foo/bar/baz/qux.txt", + "https://host.com/foo", + [("bar", "baz"), ("bar",), ()], + ), + ], +) +def test_relative_path_parents(uri, base, expected_parents_parts): + rel = UPath(uri).relative_to(UPath(base)) + parents = list(rel.parents) + assert [x.parts for x in parents] == expected_parents_parts + + +@pytest.mark.parametrize( + "protocol,pth,base", + [ + ("", "/foo/bar/baz.txt", "/foo"), + ("file", "/foo/bar/baz.txt", "/foo"), + ], +) +def test_home_works_for_local_paths(protocol, pth, base): + rel = UPath(pth, protocol=protocol).relative_to(UPath(base, protocol=protocol)) + assert rel.home() == UPath.home() + + +@pytest.mark.parametrize( + "protocol,pth,base", + [ + ("s3", "s3://bucket/foo/bar/baz.txt", "s3://bucket/foo"), + ("gcs", "gcs://bucket/foo/bar/baz.txt", "gcs://bucket/foo"), + ("memory", "memory:///foo/bar/baz.txt", "memory:///foo"), + ("https", "https://host/foo/bar/baz.txt", "https://host/foo"), + ], +) +def test_home_raises_for_non_local_paths(protocol, pth, base): + rel = UPath(pth, protocol=protocol).relative_to(UPath(base, protocol=protocol)) + with pytest.raises( + NotImplementedError, + match=re.escape(f"{type(rel).__name__}.home() is unsupported"), + ): + rel.home() + + # 'joinpath', # 'joinuri', -# 'parent', -# 'parents', # 'parser', -# 'path', -# 'protocol', -# 'storage_options', # 'with_segments', # 'resolve', # 'match', From ea2ef2718dad0e84271df7bd322de2fc45ea8211 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sun, 28 Sep 2025 13:06:38 +0200 Subject: [PATCH 14/32] tests: parser and resolve tests --- upath/tests/test_relative.py | 46 ++++++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/upath/tests/test_relative.py b/upath/tests/test_relative.py index e6ab8af1..ee167ff8 100644 --- a/upath/tests/test_relative.py +++ b/upath/tests/test_relative.py @@ -310,6 +310,7 @@ def test_relative_path_as_uri(rel_path): pytest.param(("readlink", ()), id="readlink"), pytest.param(("rename", ("a/b/c",)), id="rename"), pytest.param(("replace", ("...",)), id="replace"), + pytest.param(("resolve", ()), id="resolve"), pytest.param(("rglob", ("*.txt",)), id="rglob"), pytest.param(("rmdir", ()), id="rmdir"), pytest.param(("samefile", ("...",)), id="samefile"), @@ -511,9 +512,50 @@ def test_home_raises_for_non_local_paths(protocol, pth, base): rel.home() +@pytest.mark.parametrize( + "protocol,pth,base", + [ + ("", "/foo/bar/baz.txt", "/foo"), + ("file", "/foo/bar/baz.txt", "/foo"), + ("s3", "s3://bucket/foo/bar/baz.txt", "s3://bucket/foo"), + ("gcs", "gcs://bucket/foo/bar/baz.txt", "gcs://bucket/foo"), + ("memory", "memory:///foo/bar/baz.txt", "memory:///foo"), + ("https", "https://host/foo/bar/baz.txt", "https://host/foo"), + ], +) +def test_parser_attribute_available(protocol, pth, base): + rel_path = UPath(pth, protocol=protocol).relative_to(UPath(base, protocol=protocol)) + assert rel_path.parser is not None + + +@pytest.mark.parametrize( + "protocol", + [ + "", + "file", + ], +) +def test_relpath_path_resolve(tmp_path, protocol, monkeypatch): + """This should work for all path types that support .cwd()""" + base = UPath(tmp_path, protocol=protocol) + (base / "a" / "b").mkdir(parents=True) + (base / "a" / "b" / "file.txt").write_text("data") + monkeypatch.chdir(base) + + rel = UPath("/xyz/a/b/c/d/../../file.txt", protocol=protocol).relative_to( + UPath("/xyz", protocol=protocol) + ) + + assert str(rel) == "a/b/c/d/../../file.txt" + + resolved = rel.resolve() + assert os.fspath(resolved) == os.fspath(tmp_path / "a" / "b" / "file.txt") + assert resolved.read_text() == "data" + assert resolved.is_absolute() + assert resolved.exists() + + # 'joinpath', # 'joinuri', -# 'parser', # 'with_segments', -# 'resolve', # 'match', From 89d755e7c8031482a5bcf4a6c2635d57598c49fd Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sun, 28 Sep 2025 13:33:47 +0200 Subject: [PATCH 15/32] tests: add tests for relative path .match --- upath/tests/test_relative.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/upath/tests/test_relative.py b/upath/tests/test_relative.py index ee167ff8..28559096 100644 --- a/upath/tests/test_relative.py +++ b/upath/tests/test_relative.py @@ -555,7 +555,35 @@ def test_relpath_path_resolve(tmp_path, protocol, monkeypatch): assert resolved.exists() +@pytest.mark.parametrize( + "protocol,path,base", + [ + ("", "/foo/bar/baz/qux.txt", "/foo"), + ("file", "/foo/bar/baz/qux.txt", "/foo"), + ("s3", "s3://bucket/foo/bar/baz/qux.txt", "s3://bucket/foo"), + ("gcs", "gcs://bucket/foo/bar/baz/qux.txt", "gcs://bucket/foo"), + ("memory", "memory:///foo/bar/baz/qux.txt", "memory:///foo"), + ("https", "https://host/foo/bar/baz/qux.txt", "https://host/foo"), + ], +) +def test_relative_path_match(protocol, path, base): + """Test that match works correctly for relative paths.""" + rel = UPath(path, protocol=protocol).relative_to(UPath(base, protocol=protocol)) + + assert str(rel) == "bar/baz/qux.txt" + + # Should match patterns that match the relative path + assert rel.match("bar/baz/qux.txt") + assert rel.match("*/baz/qux.txt") + assert rel.match("bar/*/qux.txt") + assert rel.match("*/**/*.txt") # ** acts like * + + # Should not match patterns that don't match + assert not rel.match("foo/baz/qux.txt") + assert not rel.match("*.py") + assert not rel.match("other.txt") + + # 'joinpath', # 'joinuri', # 'with_segments', -# 'match', From 3db280b50445fda5dc44b74b694acd453f318340 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sun, 28 Sep 2025 14:37:37 +0200 Subject: [PATCH 16/32] tests: add tests for joinpath --- upath/tests/test_relative.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/upath/tests/test_relative.py b/upath/tests/test_relative.py index 28559096..34b8be11 100644 --- a/upath/tests/test_relative.py +++ b/upath/tests/test_relative.py @@ -584,6 +584,36 @@ def test_relative_path_match(protocol, path, base): assert not rel.match("other.txt") -# 'joinpath', +@pytest.mark.parametrize( + "protocol,path,base", + [ + ("", "/foo/bar/baz/qux.txt", "/foo"), + ("file", "/foo/bar/baz/qux.txt", "/foo"), + ("s3", "s3://bucket/foo/bar/baz/qux.txt", "s3://bucket/foo"), + ("gcs", "gcs://bucket/foo/bar/baz/qux.txt", "gcs://bucket/foo"), + ("memory", "memory:///foo/bar/baz/qux.txt", "memory:///foo"), + ("https", "https://host/foo/bar/baz/qux.txt", "https://host/foo"), + ], +) +def test_relative_path_joinpath(protocol, path, base): + """Test that joinpath works correctly for relative paths.""" + rel = UPath(path, protocol=protocol).relative_to(UPath(base, protocol=protocol)) + + # Test joining with a single segment + assert str(rel) == "bar/baz/qux.txt" + joined = rel.joinpath("extra.txt") + assert str(joined) == "bar/baz/qux.txt/extra.txt" + assert not joined.is_absolute() + + # Test joining with multiple segments + joined_multi = rel.joinpath("dir", "file.py") + assert str(joined_multi) == "bar/baz/qux.txt/dir/file.py" + assert not joined_multi.is_absolute() + + # Test that the result is still relative with same base + assert joined.protocol == joined_multi.protocol == protocol + assert joined.storage_options == joined_multi.storage_options == rel.storage_options + + # 'joinuri', # 'with_segments', From 78e7496f9275a72b92039175f34e256e0951cd86 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sun, 28 Sep 2025 13:22:12 +0200 Subject: [PATCH 17/32] upath: return dynamically created subclasses for untested protocols --- upath/registry.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/upath/registry.py b/upath/registry.py index 4ef5a7f3..50861bee 100644 --- a/upath/registry.py +++ b/upath/registry.py @@ -215,7 +215,7 @@ def get_upath_class( if not fallback: return None try: - _ = get_filesystem_class(protocol) + fs_cls = get_filesystem_class(protocol) except ValueError: return None # this is an unknown protocol else: @@ -226,4 +226,5 @@ def get_upath_class( UserWarning, stacklevel=2, ) - return upath.UPath + prefix = fs_cls.__name__.lower().removesuffix("filesystem").title() + return type(f"{prefix}Path", (upath.UPath,), {}) From ba349a0a4e762b42da877c639236d3202f66e285 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sun, 28 Sep 2025 14:40:22 +0200 Subject: [PATCH 18/32] upath: fix relative-path support for with_segments --- upath/core.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/upath/core.py b/upath/core.py index d3102265..732fe929 100644 --- a/upath/core.py +++ b/upath/core.py @@ -447,13 +447,18 @@ class UPath(_UPathMixin, OpenablePath): parser: UPathParser = LazyFlavourDescriptor() # type: ignore[assignment] def with_segments(self, *pathsegments: JoinablePathLike) -> Self: + # we change joinpath behavior if called from a relative path + # this is not fully ideal, but currently the best way to move forward + if is_relative := self._relative_base is not None: + pathsegments = (self._relative_base, *pathsegments) + new_instance = type(self)( *pathsegments, protocol=self._protocol, **self._storage_options, ) - # Preserve _relative_base if it was set - if hasattr(self, "_relative_base") and self._relative_base is not None: + + if is_relative: new_instance._relative_base = self._relative_base return new_instance From 9f30f5425ec726f9d42a488f1486ddb77e806b16 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sun, 28 Sep 2025 14:51:00 +0200 Subject: [PATCH 19/32] upath: fix __vfspath__ for relative paths --- upath/core.py | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/upath/core.py b/upath/core.py index 732fe929..413fcee3 100644 --- a/upath/core.py +++ b/upath/core.py @@ -467,23 +467,19 @@ def __str__(self) -> str: def __vfspath__(self) -> str: if self._relative_base is not None: - full_path = self._chain_parser.chain(self._chain.to_list())[0] - root_path = self._relative_base - - # Strip protocol for comparison - full_path_no_protocol = self.parser.strip_protocol(full_path) - root_path_no_protocol = self.parser.strip_protocol(root_path) - - # Calculate relative path from root to this path - if full_path_no_protocol.startswith(root_path_no_protocol): - rel_path = full_path_no_protocol[len(root_path_no_protocol) :].lstrip( - self.parser.sep + active_path = self._chain.active_path + stripped_base = self.parser.strip_protocol(self._relative_base) + if not active_path.startswith(stripped_base): + raise RuntimeError( + f"{active_path!r} is not a subpath of {stripped_base!r}" ) - return rel_path or "." - else: - # If paths don't have the expected relationship, fall back - return full_path - return self._chain_parser.chain(self._chain.to_list())[0] + + return ( + active_path.removeprefix(stripped_base).removeprefix(self.parser.sep) + or "." + ) + else: + return self._chain_parser.chain(self._chain.to_list())[0] def __repr__(self) -> str: if self._relative_base is not None: From 884a9f8051040bc23e5d3c849c4b93fb57587254 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sun, 28 Sep 2025 16:35:27 +0200 Subject: [PATCH 20/32] upath: fix parent for relative paths --- upath/core.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/upath/core.py b/upath/core.py index 413fcee3..a80f95e9 100644 --- a/upath/core.py +++ b/upath/core.py @@ -537,6 +537,24 @@ def anchor(self) -> str: return "" return self.drive + self.root + @property + def parent(self) -> Self: + if self._relative_base is not None: + if str(self) == ".": + return self + else: + # this needs to be revisited... + pth = type(self)( + self._relative_base, + str(self), + protocol=self._protocol, + **self._storage_options, + ) + parent = pth.parent + parent._relative_base = self._relative_base + return parent + return super().parent + # === ReadablePath attributes ===================================== @property From d8e0a3c2d50777ed8aabf0ccd639b074834e8e81 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sun, 28 Sep 2025 16:48:06 +0200 Subject: [PATCH 21/32] upath: .parents implementation for relative paths --- upath/core.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/upath/core.py b/upath/core.py index a80f95e9..d7baca9e 100644 --- a/upath/core.py +++ b/upath/core.py @@ -555,6 +555,19 @@ def parent(self) -> Self: return parent return super().parent + @property + def parents(self) -> Sequence[Self]: + if self._relative_base is not None: + parents = [] + parent = self + while True: + if str(parent) == ".": + break + parent = parent.parent + parents.append(parent) + return parents + return super().parents + # === ReadablePath attributes ===================================== @property From c42e84debc2527e6a963faa90f97690b195d5227 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sun, 28 Sep 2025 16:54:03 +0200 Subject: [PATCH 22/32] upath: fix FilePath.cwd() and FilePath.home() --- upath/implementations/local.py | 8 ++++++++ upath/tests/implementations/test_local.py | 22 ++++++++++++++++++++++ upath/tests/test_relative.py | 2 +- 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/upath/implementations/local.py b/upath/implementations/local.py index a2b07af7..f04fa94a 100644 --- a/upath/implementations/local.py +++ b/upath/implementations/local.py @@ -232,5 +232,13 @@ def iterdir(self) -> Iterator[Self]: def _url(self) -> SplitResult: return SplitResult._make((self.protocol, "", self.path, "", "")) + @classmethod + def cwd(cls) -> Self: + return cls(os.getcwd(), protocol="file") + + @classmethod + def home(cls) -> Self: + return cls(os.path.expanduser("~"), protocol="file") + LocalPath.register(FilePath) diff --git a/upath/tests/implementations/test_local.py b/upath/tests/implementations/test_local.py index e3f59d48..cb446bcb 100644 --- a/upath/tests/implementations/test_local.py +++ b/upath/tests/implementations/test_local.py @@ -1,3 +1,5 @@ +from pathlib import Path + import pytest from upath import UPath @@ -15,6 +17,16 @@ def path(self, local_testdir): def test_is_LocalPath(self): assert isinstance(self.path, LocalPath) + def test_cwd(self): + cwd = type(self.path).cwd() + assert isinstance(cwd, LocalPath) + assert cwd.path == Path.cwd().as_posix() + + def test_home(self): + cwd = type(self.path).home() + assert isinstance(cwd, LocalPath) + assert cwd.path == Path.home().as_posix() + @xfail_if_version("fsspec", lt="2023.10.0", reason="requires fsspec>=2023.10.0") class TestRayIOFSSpecLocal(BaseTests): @@ -25,3 +37,13 @@ def path(self, local_testdir): def test_is_LocalPath(self): assert isinstance(self.path, LocalPath) + + def test_cwd(self): + cwd = type(self.path).cwd() + assert isinstance(cwd, LocalPath) + assert cwd.path == Path.cwd().as_posix() + + def test_home(self): + cwd = type(self.path).home() + assert isinstance(cwd, LocalPath) + assert cwd.path == Path.home().as_posix() diff --git a/upath/tests/test_relative.py b/upath/tests/test_relative.py index 34b8be11..45564bb3 100644 --- a/upath/tests/test_relative.py +++ b/upath/tests/test_relative.py @@ -491,7 +491,7 @@ def test_relative_path_parents(uri, base, expected_parents_parts): ) def test_home_works_for_local_paths(protocol, pth, base): rel = UPath(pth, protocol=protocol).relative_to(UPath(base, protocol=protocol)) - assert rel.home() == UPath.home() + assert os.fspath(rel.home()) == os.fspath(UPath.home()) @pytest.mark.parametrize( From 1792ab13d27dc01a79321856dfda2579a181b832 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sun, 28 Sep 2025 17:39:11 +0200 Subject: [PATCH 23/32] upath: fix pickling of dynamically created classes --- upath/implementations/_experimental.py | 17 +++++++++++++++++ upath/registry.py | 14 +++++++++++--- 2 files changed, 28 insertions(+), 3 deletions(-) create mode 100644 upath/implementations/_experimental.py diff --git a/upath/implementations/_experimental.py b/upath/implementations/_experimental.py new file mode 100644 index 00000000..e99a02b7 --- /dev/null +++ b/upath/implementations/_experimental.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from upath.registry import get_upath_class + +if TYPE_CHECKING: + from upath import UPath + + +def __getattr__(name: str) -> type[UPath]: + if name.startswith("_") and name.endswith("Path"): + protocol = name[1:-4].lower() + cls = get_upath_class(protocol, fallback=False) + assert cls is not None + return cls + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/upath/registry.py b/upath/registry.py index 50861bee..6d0571a7 100644 --- a/upath/registry.py +++ b/upath/registry.py @@ -215,7 +215,7 @@ def get_upath_class( if not fallback: return None try: - fs_cls = get_filesystem_class(protocol) + get_filesystem_class(protocol) except ValueError: return None # this is an unknown protocol else: @@ -226,5 +226,13 @@ def get_upath_class( UserWarning, stacklevel=2, ) - prefix = fs_cls.__name__.lower().removesuffix("filesystem").title() - return type(f"{prefix}Path", (upath.UPath,), {}) + import upath.implementations._experimental as upath_experimental + + cls_name = f"_{protocol.title()}Path" + cls = type( + cls_name, + (upath.UPath,), + {"__module__": "upath.implementations._experimental"}, + ) + setattr(upath_experimental, cls_name, cls) + return cls From a4160a5935c48a62dec4f0a24b2ffe62b931e7c4 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sun, 28 Sep 2025 17:50:13 +0200 Subject: [PATCH 24/32] tests: fix .cwd and .home tests in core --- upath/tests/test_core.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/upath/tests/test_core.py b/upath/tests/test_core.py index 57be7f13..4033e616 100644 --- a/upath/tests/test_core.py +++ b/upath/tests/test_core.py @@ -64,16 +64,18 @@ def test_fsspec_compat(self): pass def test_cwd(self): - pth = type(self.path).cwd() - assert str(pth) == os.getcwd() - assert isinstance(pth, pathlib.Path) - assert isinstance(pth, UPath) + with pytest.raises( + NotImplementedError, + match=r".+Path[.]cwd\(\) is unsupported", + ): + type(self.path).cwd() def test_home(self): - pth = type(self.path).home() - assert str(pth) == os.path.expanduser("~") - assert isinstance(pth, pathlib.Path) - assert isinstance(pth, UPath) + with pytest.raises( + NotImplementedError, + match=r".+Path[.]home\(\) is unsupported", + ): + type(self.path).home() @xfail_if_version("fsspec", reason="", ge="2024.2.0") def test_iterdir_no_dir(self): From d32bb4c2b0932bee704f890f072d841b5367191f Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sun, 28 Sep 2025 18:51:06 +0200 Subject: [PATCH 25/32] upath: fix remaining relative path tests --- upath/core.py | 46 ++++++++------------------ upath/implementations/cloud.py | 8 +++++ upath/tests/implementations/test_s3.py | 2 +- upath/tests/test_relative.py | 1 + 4 files changed, 24 insertions(+), 33 deletions(-) diff --git a/upath/core.py b/upath/core.py index d7baca9e..cc3fd465 100644 --- a/upath/core.py +++ b/upath/core.py @@ -468,7 +468,9 @@ def __str__(self) -> str: def __vfspath__(self) -> str: if self._relative_base is not None: active_path = self._chain.active_path - stripped_base = self.parser.strip_protocol(self._relative_base) + stripped_base = self.parser.strip_protocol( + self._relative_base + ).removesuffix(self.parser.sep) if not active_path.startswith(stripped_base): raise RuntimeError( f"{active_path!r} is not a subpath of {stripped_base!r}" @@ -1080,6 +1082,7 @@ def relative_to( # type: ignore[override] raise NotImplementedError("walk_up=True is not implemented yet") if isinstance(other, UPath): + # revisit: ... if self.__class__ is not other.__class__: raise ValueError( "incompatible protocols:" @@ -1090,38 +1093,17 @@ def relative_to( # type: ignore[override] "incompatible storage_options:" f" {self.storage_options!r} != {other.storage_options!r}" ) + elif isinstance(other, str): + other = self.with_segments(other) + else: + raise TypeError(f"expected UPath or str, got {type(other).__name__}") - # Calculate the relative path properly - other_str = str(other) - self_str = str(self) - - # Normalize paths by ensuring root path ends with separator if it should - # Check if self starts with other as a proper path component - if self_str == other_str: - # Same path - return "." - new_instance = copy(self) - new_instance._relative_base = other_str - return new_instance - - # Check if self_str starts with other_str followed by a separator - sep = self.parser.sep - if self_str.startswith(other_str + sep): - # Valid subpath - new_instance = copy(self) - new_instance._relative_base = other_str - return new_instance - elif ( - self_str.startswith(other_str) - and len(self_str) > len(other_str) - and self_str[len(other_str)] == sep - ): - # Check if the next character is a separator - # (for cases where other_str ends with sep) - new_instance = copy(self) - new_instance._relative_base = other_str - return new_instance - - raise ValueError(f"{self_str!r} is not in the subpath of {other_str!r}") + if other not in self.parents and self != other: + raise ValueError(f"{self!s} is not in the subpath of {other!s}") + else: + rel = copy(self) + rel._relative_base = str(other) + return rel def is_relative_to(self, other, /, *_deprecated) -> bool: # type: ignore[override] if isinstance(other, UPath) and self.storage_options != other.storage_options: diff --git a/upath/implementations/cloud.py b/upath/implementations/cloud.py index d551e1a8..e1332e1a 100644 --- a/upath/implementations/cloud.py +++ b/upath/implementations/cloud.py @@ -50,6 +50,14 @@ def root(self) -> str: return "" return self.parser.sep + def __vfspath__(self): + path = super().__vfspath__() + if self._relative_base is None: + drive = self.parser.splitdrive(path)[0] + if drive and path == f"{self.protocol}://{drive}": + return f"{path}{self.root}" + return path + def mkdir( self, mode: int = 0o777, parents: bool = False, exist_ok: bool = False ) -> None: diff --git a/upath/tests/implementations/test_s3.py b/upath/tests/implementations/test_s3.py index 2029e1bd..2f8ac432 100644 --- a/upath/tests/implementations/test_s3.py +++ b/upath/tests/implementations/test_s3.py @@ -78,7 +78,7 @@ def test_no_bucket_joinpath(self, joiner): def test_creating_s3path_with_bucket(self): path = UPath("s3://", bucket="bucket", anon=self.anon, **self.s3so) - assert str(path) == "s3://bucket" + assert str(path) == "s3://bucket/" def test_iterdir_with_plus_in_name(self, s3_with_plus_chr_name): bucket, anon, s3so = s3_with_plus_chr_name diff --git a/upath/tests/test_relative.py b/upath/tests/test_relative.py index 45564bb3..c106f07a 100644 --- a/upath/tests/test_relative.py +++ b/upath/tests/test_relative.py @@ -464,6 +464,7 @@ def test_relative_path_parent(protocol, pth, base, expected_parent): ("file:///foo/bar/baz/qux.txt", "file:///foo", [("bar", "baz"), ("bar",), ()]), ("s3://bucket/foo/bar/baz/", "s3://bucket/", [("foo", "bar"), ("foo",), ()]), ("gcs://bucket/foo/bar/baz", "gcs://bucket/", [("foo", "bar"), ("foo",), ()]), + ("az://bucket/foo/bar/baz", "az://bucket/", [("foo", "bar"), ("foo",), ()]), ( "memory:///foo/bar/baz/qux.txt", "memory:///foo", From 7d24fe73b354892dcc31aa26d1a21db2bfb31b29 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sun, 28 Sep 2025 22:59:39 +0200 Subject: [PATCH 26/32] tests: fix windows path separator comparisons --- upath/tests/test_relative.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/upath/tests/test_relative.py b/upath/tests/test_relative.py index c106f07a..71019e12 100644 --- a/upath/tests/test_relative.py +++ b/upath/tests/test_relative.py @@ -492,7 +492,7 @@ def test_relative_path_parents(uri, base, expected_parents_parts): ) def test_home_works_for_local_paths(protocol, pth, base): rel = UPath(pth, protocol=protocol).relative_to(UPath(base, protocol=protocol)) - assert os.fspath(rel.home()) == os.fspath(UPath.home()) + assert rel.home().as_posix() == UPath.home().as_posix() @pytest.mark.parametrize( @@ -547,10 +547,10 @@ def test_relpath_path_resolve(tmp_path, protocol, monkeypatch): UPath("/xyz", protocol=protocol) ) - assert str(rel) == "a/b/c/d/../../file.txt" + assert rel.as_posix() == "a/b/c/d/../../file.txt" resolved = rel.resolve() - assert os.fspath(resolved) == os.fspath(tmp_path / "a" / "b" / "file.txt") + assert resolved.as_posix() == (tmp_path / "a" / "b" / "file.txt").as_posix() assert resolved.read_text() == "data" assert resolved.is_absolute() assert resolved.exists() @@ -571,7 +571,7 @@ def test_relative_path_match(protocol, path, base): """Test that match works correctly for relative paths.""" rel = UPath(path, protocol=protocol).relative_to(UPath(base, protocol=protocol)) - assert str(rel) == "bar/baz/qux.txt" + assert rel.as_posix() == "bar/baz/qux.txt" # Should match patterns that match the relative path assert rel.match("bar/baz/qux.txt") @@ -601,9 +601,9 @@ def test_relative_path_joinpath(protocol, path, base): rel = UPath(path, protocol=protocol).relative_to(UPath(base, protocol=protocol)) # Test joining with a single segment - assert str(rel) == "bar/baz/qux.txt" + assert rel.as_posix() == "bar/baz/qux.txt" joined = rel.joinpath("extra.txt") - assert str(joined) == "bar/baz/qux.txt/extra.txt" + assert joined.as_posix() == "bar/baz/qux.txt/extra.txt" assert not joined.is_absolute() # Test joining with multiple segments From cfe2228daddc32e1de027c533b5d2b6e5cfea116 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sun, 28 Sep 2025 23:08:16 +0200 Subject: [PATCH 27/32] tests: fix file uri comparison --- upath/tests/test_relative.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/upath/tests/test_relative.py b/upath/tests/test_relative.py index 71019e12..57b33b67 100644 --- a/upath/tests/test_relative.py +++ b/upath/tests/test_relative.py @@ -492,7 +492,8 @@ def test_relative_path_parents(uri, base, expected_parents_parts): ) def test_home_works_for_local_paths(protocol, pth, base): rel = UPath(pth, protocol=protocol).relative_to(UPath(base, protocol=protocol)) - assert rel.home().as_posix() == UPath.home().as_posix() + prefix = f"{protocol}://" if protocol else "" + assert rel.home().as_posix() == prefix + UPath.home().as_posix() @pytest.mark.parametrize( @@ -550,7 +551,10 @@ def test_relpath_path_resolve(tmp_path, protocol, monkeypatch): assert rel.as_posix() == "a/b/c/d/../../file.txt" resolved = rel.resolve() - assert resolved.as_posix() == (tmp_path / "a" / "b" / "file.txt").as_posix() + prefix = f"{protocol}://" if protocol else "" + assert ( + resolved.as_posix() == prefix + (tmp_path / "a" / "b" / "file.txt").as_posix() + ) assert resolved.read_text() == "data" assert resolved.is_absolute() assert resolved.exists() From 3a28728929c20cb0ced24246ccd45f9ba62ae324 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sun, 28 Sep 2025 23:13:34 +0200 Subject: [PATCH 28/32] tests: relative joinpath fix windows separator comparison --- upath/tests/test_relative.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upath/tests/test_relative.py b/upath/tests/test_relative.py index 57b33b67..b9ec1472 100644 --- a/upath/tests/test_relative.py +++ b/upath/tests/test_relative.py @@ -612,7 +612,7 @@ def test_relative_path_joinpath(protocol, path, base): # Test joining with multiple segments joined_multi = rel.joinpath("dir", "file.py") - assert str(joined_multi) == "bar/baz/qux.txt/dir/file.py" + assert joined_multi.as_posix() == "bar/baz/qux.txt/dir/file.py" assert not joined_multi.is_absolute() # Test that the result is still relative with same base From cdc0ee6f100b7bb8ee7f5d84e43a5685f144bf4a Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Sun, 28 Sep 2025 23:42:40 +0200 Subject: [PATCH 29/32] upath.implementations.cloud: fix GCSPath.exists() for gcsfs<2025.5.0 --- upath/implementations/cloud.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/upath/implementations/cloud.py b/upath/implementations/cloud.py index e1332e1a..3575e1fa 100644 --- a/upath/implementations/cloud.py +++ b/upath/implementations/cloud.py @@ -93,6 +93,13 @@ def mkdir( if "unexpected keyword argument 'create_parents'" in str(err): self.fs.mkdir(self.path) + def exists(self, *, follow_symlinks=True): + # required for gcsfs<2025.5.0, see: https://github.com/fsspec/gcsfs/pull/676 + path = self.path + if len(path) > 1: + path = path.removesuffix(self.root) + return self.fs.exists(path) + class S3Path(CloudPath): __slots__ = () From 518672544bfd93aec27a465605db278ff7dbe43f Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Mon, 29 Sep 2025 00:37:30 +0200 Subject: [PATCH 30/32] tests: add mulitple join tests for different local/fsspec combinations --- upath/tests/test_relative.py | 74 +++++++++++++++++++++++++++++++++++- 1 file changed, 72 insertions(+), 2 deletions(-) diff --git a/upath/tests/test_relative.py b/upath/tests/test_relative.py index b9ec1472..7ba484be 100644 --- a/upath/tests/test_relative.py +++ b/upath/tests/test_relative.py @@ -4,6 +4,7 @@ import pickle import re import tempfile +from pathlib import Path import pytest @@ -620,5 +621,74 @@ def test_relative_path_joinpath(protocol, path, base): assert joined.storage_options == joined_multi.storage_options == rel.storage_options -# 'joinuri', -# 'with_segments', +@pytest.mark.parametrize( + "protocol,path,base", + [ + ("", "/foo/bar/baz/qux.txt", "/foo"), + ("file", "/foo/bar/baz/qux.txt", "/foo"), + ("s3", "s3://bucket/foo/bar/baz/qux.txt", "s3://bucket/foo"), + ("gcs", "gcs://bucket/foo/bar/baz/qux.txt", "gcs://bucket/foo"), + ("memory", "memory:///foo/bar/baz/qux.txt", "memory:///foo"), + ("https", "https://host/foo/bar/baz/qux.txt", "https://host/foo"), + ], +) +def test_join_local_absolute_path_to_relative(protocol, path, base, tmp_path): + """Test that joining an absolute path to a relative path works correctly.""" + rel = UPath(path, protocol=protocol).relative_to(base) + + assert rel.as_posix() == "bar/baz/qux.txt" + tmp_path.joinpath("bar/baz/qux.txt").parent.mkdir(parents=True, exist_ok=True) + tmp_path.joinpath("bar/baz/qux.txt").write_text("data") + + assert UPath(tmp_path).joinpath(rel).read_text() == "data" + + +@pytest.mark.parametrize( + "protocol,path", + [ + ("", "/foo/bar"), + ("file", "/foo/bar"), + ("s3", "s3://bucket/foo/bar"), + ("gcs", "gcs://bucket/foo/bar"), + ("memory", "memory:///foo/bar"), + ("https", "https://host/foo/bar"), + ], +) +def test_join_fsspec_absolute_path_to_relative(protocol, path): + p = UPath(path, protocol=protocol) + + x = p.joinpath(Path("a/b/c")) + assert x.path.endswith("foo/bar/a/b/c") + + +@pytest.mark.parametrize( + "proto0,path0", + [ + ("", "/foo/bar"), + ("file", "/foo/bar"), + ("s3", "s3://bucket/foo/bar"), + ("gcs", "gcs://bucket/foo/bar"), + ("memory", "memory:///foo/bar"), + ("https", "https://host/foo/bar"), + ], +) +@pytest.mark.parametrize( + "proto1,path1,base1", + [ + ("", "/foo/bar", "/foo"), + ("file", "/foo/bar", "/foo"), + ("s3", "s3://bucket/foo/bar", "s3://bucket/foo"), + ("gcs", "gcs://bucket/foo/bar", "gcs://bucket/foo"), + ("memory", "memory:///foo/bar", "memory:///foo"), + ("https", "https://host/foo/bar", "https://host/foo"), + ], +) +def test_join_fsspec_absolute_path_to_fsspec_relative( + proto0, path0, proto1, path1, base1 +): + p0 = UPath(path0, protocol=proto0) + p1 = UPath(path1, protocol=proto1).relative_to(base1) + assert str(p1) == "bar" + + x = p0.joinpath(p1) + assert x.path.endswith("foo/bar/bar") From 6998a2d098a188f0c2dc41e7f2ebc955d0c967d8 Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Mon, 29 Sep 2025 00:38:52 +0200 Subject: [PATCH 31/32] upath: fix joinpath for relative upaths --- upath/_flavour.py | 7 +++++-- upath/_protocol.py | 5 +++++ upath/implementations/local.py | 6 +++--- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/upath/_flavour.py b/upath/_flavour.py index e6b41054..ed107c60 100644 --- a/upath/_flavour.py +++ b/upath/_flavour.py @@ -17,6 +17,7 @@ from fsspec.registry import registry as _class_registry from fsspec.spec import AbstractFileSystem +import upath from upath._flavour_sources import FileSystemFlavourBase from upath._flavour_sources import flavour_registry from upath._protocol import get_upath_protocol @@ -239,12 +240,14 @@ def local_file(self) -> bool: def stringify_path(pth: PathOrStr) -> str: if isinstance(pth, str): out = pth + elif isinstance(pth, upath.UPath) and not pth.is_absolute(): + out = str(pth) elif getattr(pth, "__fspath__", None) is not None: assert hasattr(pth, "__fspath__") out = pth.__fspath__() elif isinstance(pth, os.PathLike): out = str(pth) - elif hasattr(pth, "path"): # type: ignore[unreachable] + elif isinstance(pth, upath.UPath) and pth.is_absolute(): out = pth.path else: out = str(pth) @@ -288,7 +291,7 @@ def join(self, path: PathOrStr, *paths: PathOrStr) -> str: if self.local_file: return os.path.join( self.strip_protocol(path), - *paths, # type: ignore[arg-type] + *map(self.stringify_path, paths), ) if self.netloc_is_anchor: drv, p0 = self.splitdrive(path) diff --git a/upath/_protocol.py b/upath/_protocol.py index db1fead1..51588c1d 100644 --- a/upath/_protocol.py +++ b/upath/_protocol.py @@ -105,7 +105,12 @@ def compatible_protocol( *args: str | os.PathLike[str] | PurePath | JoinablePath, ) -> bool: """check if UPath protocols are compatible""" + from upath.core import UPath + for arg in args: + if isinstance(arg, UPath) and not arg.is_absolute(): + # relative UPath are always compatible + continue other_protocol = get_upath_protocol(arg) # consider protocols equivalent if they match up to the first "+" other_protocol = other_protocol.partition("+")[0] diff --git a/upath/implementations/local.py b/upath/implementations/local.py index f04fa94a..3e2bb5d9 100644 --- a/upath/implementations/local.py +++ b/upath/implementations/local.py @@ -165,17 +165,17 @@ def _url(self) -> SplitResult: def joinpath(self, *other) -> Self: if not compatible_protocol("", *other): raise ValueError("can't combine incompatible UPath protocols") - return super().joinpath(*other) + return super().joinpath(*map(str, other)) def __truediv__(self, other) -> Self: if not compatible_protocol("", other): raise ValueError("can't combine incompatible UPath protocols") - return super().__truediv__(other) + return super().__truediv__(str(other)) def __rtruediv__(self, other) -> Self: if not compatible_protocol("", other): raise ValueError("can't combine incompatible UPath protocols") - return super().__rtruediv__(other) + return super().__rtruediv__(str(other)) UPath.register(LocalPath) From 866af9411fdc11698da6cad84725f4519ee9eb3a Mon Sep 17 00:00:00 2001 From: Andreas Poehlmann Date: Mon, 29 Sep 2025 00:47:21 +0200 Subject: [PATCH 32/32] upath: fix local joinpath stringification --- upath/implementations/local.py | 19 ++++++++++++++++--- upath/tests/test_relative.py | 2 +- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/upath/implementations/local.py b/upath/implementations/local.py index 3e2bb5d9..d11a2656 100644 --- a/upath/implementations/local.py +++ b/upath/implementations/local.py @@ -165,17 +165,30 @@ def _url(self) -> SplitResult: def joinpath(self, *other) -> Self: if not compatible_protocol("", *other): raise ValueError("can't combine incompatible UPath protocols") - return super().joinpath(*map(str, other)) + return super().joinpath( + *( + str(o) if isinstance(o, UPath) and not o.is_absolute() else o + for o in other + ) + ) def __truediv__(self, other) -> Self: if not compatible_protocol("", other): raise ValueError("can't combine incompatible UPath protocols") - return super().__truediv__(str(other)) + return super().__truediv__( + str(other) + if isinstance(other, UPath) and not other.is_absolute() + else other + ) def __rtruediv__(self, other) -> Self: if not compatible_protocol("", other): raise ValueError("can't combine incompatible UPath protocols") - return super().__rtruediv__(str(other)) + return super().__rtruediv__( + str(other) + if isinstance(other, UPath) and not other.is_absolute() + else other + ) UPath.register(LocalPath) diff --git a/upath/tests/test_relative.py b/upath/tests/test_relative.py index 7ba484be..fef84a22 100644 --- a/upath/tests/test_relative.py +++ b/upath/tests/test_relative.py @@ -657,7 +657,7 @@ def test_join_local_absolute_path_to_relative(protocol, path, base, tmp_path): def test_join_fsspec_absolute_path_to_relative(protocol, path): p = UPath(path, protocol=protocol) - x = p.joinpath(Path("a/b/c")) + x = p.joinpath(Path("a/b/c").as_posix()) assert x.path.endswith("foo/bar/a/b/c")