From e3bfb0cba78757375db0d6d51994fcf3a661588b Mon Sep 17 00:00:00 2001 From: laughingman7743 Date: Tue, 17 Nov 2026 03:05:09 +0900 Subject: [PATCH 1/2] Act on listed identities in pairing and copy/mv/get (GH-1403) copy(), mv() and get(), on both filesystems, now act on the keys that a listing returns by their buckets and keys. A key such as a/x?versionId=null was parsed again as the null version of a/x, so a recursive copy, move or download raised that two paths had the same destination, copied the version, or wrote both objects to one file. - S3PathPairing accepts S3Path sources, names them after their bucket and key as they are, and compares the paths of moves and copies as S3Path objects; strings are paired, compared and returned as before. s3_copy_pairs() returns the S3 destination of an S3Path source as the parsed destination root plus the literal suffix of the source, and rejects a destination root that names a version. - The filesystems pair the expansion of _expand_paths(), whose listed entries are S3Path objects, and look the directories and conflict candidates up by them. - cp_file(), get_file(), S3File and S3FileTarget.resolve() accept an S3Path, which they look up and read by its bucket, key and version. - The async _copy() pairs every copy as the sync copy() does, and the async _get() runs the downloads as fsspec does for lists of paths, with the branches of the callback named after the sources as strings. Co-Authored-By: Claude Opus 5.5 --- docs/filesystem.md | 16 +- pyathena/filesystem/s3.py | 201 ++++++++++++----- pyathena/filesystem/s3_async.py | 120 +++++++--- pyathena/filesystem/s3_path_pairing.py | 207 +++++++++++++----- tests/pyathena/filesystem/test_s3.py | 80 ++++++- tests/pyathena/filesystem/test_s3_async.py | 60 +++++ .../filesystem/test_s3_path_pairing.py | 86 ++++++++ tests/pyathena/util.py | 34 +++ 8 files changed, 641 insertions(+), 163 deletions(-) diff --git a/docs/filesystem.md b/docs/filesystem.md index 85dcd236f..74c14d359 100644 --- a/docs/filesystem.md +++ b/docs/filesystem.md @@ -903,11 +903,17 @@ except Exception: ## Path pairing -`S3PathPairing` pairs the paths of one `copy()`, `get()` or `mv()`: `mv()` always, and -`copy()` and `get()` when a source has a version ID and the destination is one path -(fsspec pairs the others). Its `delete_paths()` splits the paths of an `rm()` into -those deleted as given and those expanded. The pairing is fsspec's, except that a -path with a version ID names that version, and its destination is named after its key. +`S3PathPairing` pairs the paths of one `copy()`, `get()` or `mv()`. Its `delete_paths()` +splits the paths of an `rm()` into those deleted as given and those expanded. The +pairing is fsspec's, except that a path with a version ID names that version, and its +destination is named after its key. + +A source may also be an `S3Path`, as the filesystems pass the keys that their listings +return. It names its bucket and key as they are, also a key that ends in what looks +like a version ID query, such as `a/x?versionId=null`, and its destination is named +after that key. `s3_copy_pairs()` pairs the sources as `copy_pairs()` does, and returns +the S3 destination of an `S3Path` source as an `S3Path`. `conflict_candidates()` returns +the candidates of `S3Path` sources as `S3Path` objects, and `missing` accepts them. A pairing is a frozen dataclass of `path1`, `path2`, `recursive` and `maxdepth`, and holds no filesystem. The filesystems read from it what to look up (`expands`, diff --git a/pyathena/filesystem/s3.py b/pyathena/filesystem/s3.py index 35fccee9b..ebaa4798b 100644 --- a/pyathena/filesystem/s3.py +++ b/pyathena/filesystem/s3.py @@ -2752,7 +2752,7 @@ def mv(self, path1, path2, recursive=False, maxdepth=None, batch_size=None, **kw concurrency = min(limit, len(pairs)) call_kwargs = self._batch_kwargs(concurrency, kwargs) - def copy_pair(pair: tuple[str, str]) -> bool: + def copy_pair(pair: tuple[str | S3Path, str | S3Path]) -> bool: """Copy a pair of the move. Args: @@ -2777,7 +2777,7 @@ def _move_pairs( recursive: bool = False, maxdepth: int | None = None, source_params: Mapping[str, Any] | None = None, - ) -> list[tuple[str, str]]: + ) -> list[tuple[str | S3Path, str | S3Path]]: """Pair and check the paths of a move (see :meth:`S3PathPairing.move_pairs`). Args: @@ -2807,10 +2807,10 @@ def _move_pairs( def _conflict_lookups( self, pairing: S3PathPairing, - pairs: list[tuple[str, str]], + pairs: list[tuple[str | S3Path, str | S3Path]], source_params: Mapping[str, Any] | None = None, copy: bool = False, - ) -> tuple[set[str], set[str]]: + ) -> tuple[set[str | S3Path], set[str]]: """Look up what the conflict checks of a move or a copy need. See :meth:`S3PathPairing.conflict_candidates`. @@ -2831,11 +2831,11 @@ def _conflict_lookups( Raises: OSError: If bucket versioning cannot be read. """ - paths = {p: S3Path.parse(p) for pair in pairs for p in pair} - unversioned = {p.name for p in paths.values() if not p.version_id} + paths = {self._pair_path(p) for pair in pairs for p in pair} + unversioned = {p.name for p in paths if not p.version_id} buckets = { p.bucket - for p in paths.values() + for p in paths if p.version_id == "null" and p.name in unversioned and not p.is_directory_bucket } versioned_buckets = { @@ -2847,44 +2847,86 @@ def _conflict_lookups( for source in pairing.conflict_candidates( pairs, versioning_enabled_buckets=versioned_buckets, copy=copy ) - if self._head_object(S3Path.parse(source), lookup_kwargs=lookup_kwargs) is None + if self._head_object(self._pair_path(source), lookup_kwargs=lookup_kwargs) is None } return missing, versioned_buckets + @staticmethod + def _pair_path(path: str | S3Path) -> S3Path: + """Return a path of the pairs of a copy as an ``S3Path``. + + Args: + path: A path string that the caller passed, which is parsed, or + a listed path. + + Returns: + The path. + """ + return path if isinstance(path, S3Path) else S3Path.parse(path) + def _copy_pairs( self, pairing: S3PathPairing, - isdir: Callable[[str], bool] | None = None, + local: bool = False, **kwargs: Any, - ) -> list[tuple[str, str]]: - """Expand and pair the paths of a copy (see :meth:`S3PathPairing.copy_pairs`). + ) -> list[tuple[str | S3Path, str | S3Path]]: + """Expand and pair the paths of a copy (see :meth:`S3PathPairing.s3_copy_pairs`). - The destination is looked up only when it decides the pairing. + The sources are expanded as :meth:`_expand_paths` names them: a path + that the caller passes as the string it was given, and a listed entry + as its bucket and key. The destination is looked up only when it + decides the pairing. Args: pairing: The paths of the copy. - isdir: Whether the destination is a directory, by default - ``self.isdir``; ``get()`` passes the local filesystem's. - **kwargs: Passed to ``expand_path()``, as fsspec's ``get()`` passes - its keyword arguments. + local: Whether the destinations are the local paths of ``get()``, + which the local filesystem looks up, and which are named as + :meth:`S3PathPairing.copy_pairs` names them. + **kwargs: Passed to the expansion, as fsspec's ``get()`` passes its + keyword arguments to ``expand_path()``. Returns: The sources and their destinations. """ if not pairing.expands: - return pairing.copy_pairs() - sources = self.expand_path( + return list(pairing.copy_pairs()) + expanded = self._expand_paths( pairing.path1, recursive=pairing.recursive, maxdepth=pairing.maxdepth, **kwargs ) if pairing.skips_directories: - sources = [p for p in sources if not (trailing_sep(p) or self.isdir(p))] + expanded = [ + (name, target) + for name, target in expanded + if not (trailing_sep(name) or self._target_isdir(target)) + ] + sources = [target for _, target in expanded] + isdir = LocalFileSystem().isdir if local else self.isdir destination_is_dir = ( # A string, as looks_up_destination checks. - (isdir or self.isdir)(cast(str, pairing.path2)) - if sources and pairing.looks_up_destination - else None + isdir(cast(str, pairing.path2)) if sources and pairing.looks_up_destination else None ) - return pairing.copy_pairs(sources, destination_is_dir) + if local: + return list(pairing.copy_pairs(sources, destination_is_dir)) + return pairing.s3_copy_pairs(sources, destination_is_dir) + + def _target_isdir(self, target: str | S3Path) -> bool: + """Return whether a path that an expansion names is a directory. + + Args: + target: A path string that the caller passed, which ``isdir()`` + looks up, or a listed path, looked up by its bucket and key. + + Returns: + Whether the path is a directory. + """ + if isinstance(target, str): + return bool(self.isdir(target)) + try: + return bool( + self._path_info(target).get("type") == S3ObjectType.S3_OBJECT_TYPE_DIRECTORY + ) + except OSError: + return False def copy( self, @@ -2951,7 +2993,7 @@ def copy( concurrency = min(limit, len(pairs)) call_kwargs = self._batch_kwargs(concurrency, kwargs) - def copy_pair(pair: tuple[str, str]) -> None: + def copy_pair(pair: tuple[str | S3Path, str | S3Path]) -> None: """Copy a pair. Args: @@ -3011,7 +3053,7 @@ def get( root = make_path_posix(lpath) if isinstance(lpath, (str, os.PathLike)) else lpath pairs = self._copy_pairs( S3PathPairing(rpath, root, recursive=recursive, maxdepth=maxdepth), - isdir=LocalFileSystem().isdir, + local=True, **kwargs, ) if isinstance(root, str): @@ -3026,14 +3068,15 @@ def get( call_kwargs = self._batch_kwargs(concurrency, kwargs) locked = _S3LockedCallback(callback, threading.Lock()) - def get_pair(pair: tuple[str, Any]) -> None: + def get_pair(pair: tuple[str | S3Path, Any]) -> None: """Download a pair with a branch of the callback. Args: - pair: The S3 source and its local destination. + pair: The S3 source and its local destination. The branch + is named after the source as a string. """ rp, lp = pair - with locked.branched(rp, lp) as child: + with locked.branched(str(rp), lp) as child: self.get_file(rp, lp, callback=child, **call_kwargs) locked.relative_update(1) @@ -3082,8 +3125,8 @@ def _check_local_destinations(destinations: list[Any]) -> None: def cp_file( self, - path1: str | os.PathLike[str], - path2: str | os.PathLike[str], + path1: str | os.PathLike[str] | S3Path, + path2: str | os.PathLike[str] | S3Path, recursive=False, maxdepth=None, on_error=None, @@ -3096,8 +3139,10 @@ def cp_file( and multipart copy based on object size. Args: - path1: Source S3 path (s3://bucket/key). - path2: Destination S3 path (s3://bucket/key). + path1: Source S3 path (s3://bucket/key), or an ``S3Path``, whose + bucket, key and version are used as they are, such as a key + that a listing returns. + path2: Destination S3 path (s3://bucket/key), or an ``S3Path``. recursive: Unused parameter for fsspec compatibility. maxdepth: Unused parameter for fsspec compatibility. on_error: Unused parameter for fsspec compatibility. @@ -3125,12 +3170,12 @@ def cp_file( path1, path2 = stringify_path(path1), stringify_path(path2) self._copy_file(path1, path2, **kwargs) - def _copy_file(self, path1: str, path2: str, **kwargs) -> bool: + def _copy_file(self, path1: str | S3Path, path2: str | S3Path, **kwargs) -> bool: """Copy an S3 object as :meth:`cp_file` does. Args: - path1: Source S3 path (s3://bucket/key). - path2: Destination S3 path (s3://bucket/key). + path1: Source S3 path (s3://bucket/key), or an ``S3Path``. + path2: Destination S3 path (s3://bucket/key), or an ``S3Path``. **kwargs: Additional S3 copy parameters, as for :meth:`cp_file`. Returns: @@ -3143,8 +3188,8 @@ def _copy_file(self, path1: str, path2: str, **kwargs) -> bool: # Parameters of the multipart copy, not of the S3 requests. block_size = kwargs.pop("block_size", None) max_workers = kwargs.pop("max_workers", None) - source = S3Path.parse(path1) - destination = S3Path.parse(path2) + source = self._pair_path(path1) + destination = self._pair_path(path2) if destination.version_id: raise ValueError("Cannot copy to a versioned file.") if not source.key or not destination.key: @@ -3152,7 +3197,12 @@ def _copy_file(self, path1: str, path2: str, **kwargs) -> bool: # Looked up with the parameters that authorize the access to the # source, such as its customer-provided key. - info1 = self.info(path1, **self.core.copy_source_params(kwargs)) + source_params = self.core.copy_source_params(kwargs) + info1 = ( + self.info(path1, **source_params) + if isinstance(path1, str) + else self._path_info(source, lookup_kwargs=self._get_lookup_kwargs(source_params)) + ) if info1.get("type") == S3ObjectType.S3_OBJECT_TYPE_DIRECTORY: # Recursive copy() passes the directories too; S3 has no object # to copy for them. @@ -3173,7 +3223,10 @@ def _copy_file(self, path1: str, path2: str, **kwargs) -> bool: finally: # A multipart copy that fails to copy an annotation has already # written the destination. - self.invalidate_cache(path2) + if isinstance(path2, str): + self.invalidate_cache(path2) + else: + self._invalidate(destination) return True def _copy_object_with_multipart_upload( @@ -3855,7 +3908,7 @@ def put_file( def get_file( self, - rpath: str | os.PathLike[str], + rpath: str | os.PathLike[str] | S3Path, lpath=None, callback=_DEFAULT_CALLBACK, outfile=None, @@ -3883,7 +3936,9 @@ def get_file( directories of a local file ``lpath`` are created as needed. Args: - rpath: S3 source path (s3://bucket/key). + rpath: S3 source path (s3://bucket/key), or an ``S3Path``, whose + bucket, key and version are used as they are, such as a key + that a listing returns. lpath: Local destination path, or a file-like object to write to. Not needed when ``outfile`` is given. callback: Progress callback for tracking download progress, with @@ -3899,13 +3954,13 @@ def get_file( ``DEFAULT_BLOCK_SIZE`` if that is not positive either, and the size of the callback is unknown. """ - path_version_id = S3Path.parse(self._strip_protocol(rpath)).version_id + s3_path = rpath if isinstance(rpath, S3Path) else S3Path.parse(self._strip_protocol(rpath)) if outfile is None and isfilelike(lpath): outfile = lpath elif ( outfile is None - and not (path_version_id or kwargs.get("version_id")) - and self.isdir(rpath) + and not (s3_path.version_id or kwargs.get("version_id")) + and (self.isdir(rpath) if not isinstance(rpath, S3Path) else self._target_isdir(rpath)) ): # A requested version always names an object, while isdir() # would look up the latest version, or the prefix of the same @@ -3915,9 +3970,7 @@ def get_file( # The codec, if any, as fsspec's open() resolves it, with "infer" # from the key without the version. - codec = get_compression( - S3Path.parse(self._strip_protocol(rpath)).key or "", kwargs.pop("compression", None) - ) + codec = get_compression(s3_path.key or "", kwargs.pop("compression", None)) # Not the caller's cache: one such as "all" would read the whole # object when the file is opened, and "background" reads outside # the stop scopes of the caller. Without a codec, the object is read @@ -3934,7 +3987,11 @@ def get_file( # The remote file is opened first so that no local file is created # when open() finds no object at the path. with contextlib.ExitStack() as stack: - remote = stack.enter_context(self.open(rpath, "rb", **kwargs)) + remote = stack.enter_context( + self._open(rpath, "rb", **kwargs) + if isinstance(rpath, S3Path) + else self.open(rpath, "rb", **kwargs) + ) reader: Any = None if codec is not None: # As fsspec's open() wraps the file, but closed with it. @@ -4495,7 +4552,7 @@ def _key_exists(self, path: S3Path, lookup_kwargs: Mapping[str, Any]) -> bool: return False return True - def _object_exists(self, path: str, lookup_kwargs: Mapping[str, Any]) -> bool: + def _object_exists(self, path: str | S3Path, lookup_kwargs: Mapping[str, Any]) -> bool: """Check whether an object exists at the path, for a write that must not replace one. A key that is only a prefix of other keys has no object, as when @@ -4510,7 +4567,11 @@ def _object_exists(self, path: str, lookup_kwargs: Mapping[str, Any]) -> bool: True if an object exists at the path. """ try: - info = self.info(path, **lookup_kwargs) + info = ( + self.info(path, **lookup_kwargs) + if isinstance(path, str) + else self._path_info(path, lookup_kwargs=lookup_kwargs) + ) except FileNotFoundError: return False return bool(info.get("type") != S3ObjectType.S3_OBJECT_TYPE_DIRECTORY) @@ -4660,7 +4721,7 @@ def _ls_from_cache(self, path: S3Path) -> list[S3Object] | S3Object | None: def _open( self, - path: str, + path: str | S3Path, mode: str = "rb", block_size: int | None = None, cache_type: str | None = None, @@ -4777,7 +4838,7 @@ class S3FileTarget: def resolve( cls, fs: S3FileSystem, - path: str, + path: str | S3Path, mode: str, *, block_size: int, @@ -4791,7 +4852,10 @@ def resolve( Args: fs: The filesystem that the file belongs to. - path: S3 path (s3://bucket/key) of the file. + path: S3 path (s3://bucket/key) of the file, or an ``S3Path``, + whose bucket, key and version are used as they are, such as + a key that a listing returns, and which is looked up by them. + The path of the file is then the string of the ``S3Path``. mode: The file mode: ``rb``, ``wb``, ``ab``, or ``xb``. block_size: The block size of the file. version_id: The version ID to read. Must match the version ID in @@ -4816,7 +4880,11 @@ def resolve( """ # A new dictionary, so that the caller's is not modified. kwargs: dict[str, Any] = dict(request_kwargs or {}) - s3_path = S3Path.parse(path) + # A path string is looked up as a string, and an S3Path by its bucket, + # key and version. + typed = path if isinstance(path, S3Path) else None + s3_path = typed or S3Path.parse(cast(str, path)) + path = str(path) path_version_id = s3_path.version_id if not s3_path.key: raise ValueError("The path does not contain a key.") @@ -4856,7 +4924,11 @@ def resolve( if "r" in mode: # Looked up here; the base class initializer of a file would # otherwise take the size from the latest version of the object. - info = fs.info(path, version_id=version_id, **lookup_kwargs) + info = ( + fs._path_info(typed.with_version_id(version_id), lookup_kwargs=lookup_kwargs) + if typed + else fs.info(path, version_id=version_id, **lookup_kwargs) + ) if info.get("type") == S3ObjectType.S3_OBJECT_TYPE_DIRECTORY: # A prefix has no object to read. raise FileNotFoundError(path) @@ -4880,7 +4952,11 @@ def resolve( # The rewritten object keeps the metadata of the existing one, # which a cached listing entry lacks, so look up the object. with contextlib.suppress(FileNotFoundError): - appended = fs.info(path, refresh=True, **lookup_kwargs) + appended = ( + fs._path_info(typed, refresh=True, lookup_kwargs=lookup_kwargs) + if typed + else fs.info(path, refresh=True, **lookup_kwargs) + ) if ( appended is not None and appended.get("type") == S3ObjectType.S3_OBJECT_TYPE_DIRECTORY @@ -4896,7 +4972,11 @@ def resolve( # the whole object is read, and an object replaced since # is not. condition = {"IfMatch": etag} if etag else {} - append_data = fs.cat_file(path, **condition, **lookup_kwargs) + append_data = ( + fs._cat_object(typed, **condition, **lookup_kwargs) + if typed + else fs.cat_file(path, **condition, **lookup_kwargs) + ) # The attributes of the object looked up are kept where the # caller gives none: the encryption as a whole, so that its # fields are not combined with another encryption, and @@ -4919,7 +4999,7 @@ def resolve( elif "x" in mode: # Checked up front so that no data is uploaded for an existing # object, and on commit with IfNoneMatch for one created since. - if fs._object_exists(path, lookup_kwargs): + if fs._object_exists(typed or path, lookup_kwargs): raise FileExistsError(path) kwargs.update({"IfNoneMatch": "*"}) return cls( @@ -4947,7 +5027,7 @@ class S3File(AbstractBufferedFile): def __init__( self, fs: S3FileSystem, - path: str, + path: str | S3Path, mode: str = "rb", version_id: str | None = None, max_workers: int | None = None, @@ -4974,7 +5054,8 @@ def __init__( Args: fs: The filesystem that the file belongs to. - path: S3 path (s3://bucket/key) of the file. + path: S3 path (s3://bucket/key) of the file, or an ``S3Path`` + (see :meth:`S3FileTarget.resolve`). mode: The file mode: ``rb``, ``wb``, ``ab``, or ``xb``. version_id: The version ID to read. Must match the version ID in the path if both are given. A version cannot be given, in diff --git a/pyathena/filesystem/s3_async.py b/pyathena/filesystem/s3_async.py index 9dde3240c..30f19331a 100644 --- a/pyathena/filesystem/s3_async.py +++ b/pyathena/filesystem/s3_async.py @@ -32,7 +32,7 @@ ) from fsspec.callbacks import _DEFAULT_CALLBACK from fsspec.core import get_compression -from fsspec.implementations.local import LocalFileSystem, make_path_posix +from fsspec.implementations.local import make_path_posix from fsspec.utils import check_contained, stringify_path from pyathena.aio.util import create_in_thread @@ -623,7 +623,7 @@ def _put_file_in_transaction( async def _get_file( self, - rpath: str | os.PathLike[str], + rpath: str | os.PathLike[str] | S3Path, lpath: str | os.PathLike[str], callback=_DEFAULT_CALLBACK, **kwargs, @@ -824,11 +824,11 @@ async def _copy( ) -> None: """Copy files within S3. - See :meth:`S3FileSystem.copy`. The copies run as in fsspec's - ``_copy()``, ``batch_size`` of them at once as tasks. A cancellation - cancels the copies and waits for them; a copy that is already being - cancelled, such as by the shutdown of the event loop, is not - cancelled again. + See :meth:`S3FileSystem.copy`. The paths are paired as there, and the + copies run as in fsspec's ``_copy()``, ``batch_size`` of them at once + as tasks. A cancellation cancels the copies and waits for them; a + copy that is already being cancelled, such as by the shutdown of the + event loop, is not cancelled again. Args: path1: Source S3 path, glob pattern, or list of them. @@ -841,19 +841,19 @@ async def _copy( **kwargs: Additional S3 copy parameters passed to ``_cp_file()``. """ path1, path2 = self._sync_fs._stringify_paths(path1), self._sync_fs._stringify_paths(path2) - sources = [path1] if isinstance(path1, str) else path1 - if isinstance(path2, str) and any(S3Path.has_version_id(p) for p in sources): - pairs = await self._thread_pool.run( - self._sync_fs._copy_pairs, - S3PathPairing(path1, path2, recursive=recursive, maxdepth=maxdepth), - ) - if not pairs: - return - path1, path2 = [p1 for p1, _ in pairs], [p2 for _, p2 in pairs] + # Paired as the sync copy() pairs them, so that the keys that a + # listing returns are copied by their buckets and keys; fsspec copies + # the lists of the pairs. + pairs = await self._thread_pool.run( + self._sync_fs._copy_pairs, + S3PathPairing(path1, path2, recursive=recursive, maxdepth=maxdepth), + ) + if not pairs: + return await _S3BatchTasks().run( super()._copy( - path1, - path2, + [p1 for p1, _ in pairs], + [p2 for _, p2 in pairs], recursive=recursive, on_error=on_error, maxdepth=maxdepth, @@ -901,7 +901,7 @@ async def _get( pairs = await self._thread_pool.run( self._sync_fs._copy_pairs, S3PathPairing(rpath, root, recursive=recursive, maxdepth=maxdepth), - isdir=LocalFileSystem().isdir, + local=True, ) if isinstance(root, str): check_contained(root, [lp for _, lp in pairs]) @@ -915,12 +915,47 @@ async def _get( if batch_size is not None and batch_size > 0: concurrency = min(concurrency, batch_size) await _S3BatchTasks().run( - super()._get( - [rp for rp, _ in pairs], - [lp for _, lp in pairs], - callback=callback, - **self._sync_fs._batch_kwargs(concurrency, kwargs), - ) + self._get_pairs(pairs, callback, **self._sync_fs._batch_kwargs(concurrency, kwargs)) + ) + + async def _get_pairs( + self, pairs: list[tuple[str | S3Path, Any]], callback: Any, **kwargs + ) -> list[Any]: + """Download the pairs of a ``get()`` as fsspec's ``_get()`` downloads lists of paths. + + Each download gets a branch of the callback named after the string of + its source, and the sources that a listing returns are downloaded by + their buckets and keys. + + Args: + pairs: The S3 sources and their local destinations. + callback: Progress callback. + **kwargs: Passed to ``_get_file()``, and ``batch_size``, the + number of downloads to run at once. + + Returns: + The results of the downloads, as fsspec's ``_get()`` returns them. + """ + for _, lp in pairs: + os.makedirs(os.path.dirname(lp), exist_ok=True) + batch_size = kwargs.pop("batch_size", self.batch_size) + callback.set_size(len(pairs)) + + async def get_pair(rp: str | S3Path, lp: Any) -> None: + """Download a pair with a branch of the callback. + + Args: + rp: The S3 source. + lp: The local destination. + """ + with callback.branched(str(rp), lp, **kwargs) as child: + return await self._get_file(rp, lp, callback=child, **kwargs) + + return cast( + list[Any], + await _run_coros_in_chunks( + [get_pair(rp, lp) for rp, lp in pairs], batch_size=batch_size, callback=callback + ), ) async def _put( @@ -1150,8 +1185,8 @@ async def _cat_ranges( async def _cp_file( self, - path1: str | os.PathLike[str], - path2: str | os.PathLike[str], + path1: str | os.PathLike[str] | S3Path, + path2: str | os.PathLike[str] | S3Path, recursive=False, maxdepth=None, on_error=None, @@ -1163,8 +1198,9 @@ async def _cp_file( the files under it, is skipped. Args: - path1: Source S3 path (s3://bucket/key). - path2: Destination S3 path (s3://bucket/key). + path1: Source S3 path (s3://bucket/key), or an ``S3Path``, as + for :meth:`S3FileSystem.cp_file`. + path2: Destination S3 path (s3://bucket/key), or an ``S3Path``. recursive: Unused parameter for fsspec compatibility. maxdepth: Unused parameter for fsspec compatibility. on_error: Unused parameter for fsspec compatibility. @@ -1178,12 +1214,12 @@ async def _cp_file( path1, path2 = stringify_path(path1), stringify_path(path2) await self._copy_file(path1, path2, **kwargs) - async def _copy_file(self, path1: str, path2: str, **kwargs) -> bool: + async def _copy_file(self, path1: str | S3Path, path2: str | S3Path, **kwargs) -> bool: """Copy an S3 object as :meth:`_cp_file` does. Args: - path1: Source S3 path (s3://bucket/key). - path2: Destination S3 path (s3://bucket/key). + path1: Source S3 path (s3://bucket/key), or an ``S3Path``. + path2: Destination S3 path (s3://bucket/key), or an ``S3Path``. **kwargs: Additional S3 copy parameters, as for :meth:`_cp_file`. Returns: @@ -1196,8 +1232,8 @@ async def _copy_file(self, path1: str, path2: str, **kwargs) -> bool: # Parameters of the multipart copy, not of the S3 requests. block_size = kwargs.pop("block_size", None) max_workers = kwargs.pop("max_workers", None) - source = S3Path.parse(path1) - destination = S3Path.parse(path2) + source = self._sync_fs._pair_path(path1) + destination = self._sync_fs._pair_path(path2) if destination.version_id: raise ValueError("Cannot copy to a versioned file.") if not source.key or not destination.key: @@ -1205,7 +1241,16 @@ async def _copy_file(self, path1: str, path2: str, **kwargs) -> bool: # Looked up with the parameters that authorize the access to the # source; see S3FileSystem._copy_file. - info1 = await self._info(path1, **self.core.copy_source_params(kwargs)) + source_params = self.core.copy_source_params(kwargs) + info1 = ( + await self._info(path1, **source_params) + if isinstance(path1, str) + else await self._thread_pool.run( + self._sync_fs._path_info, + source, + lookup_kwargs=self._sync_fs._get_lookup_kwargs(source_params), + ) + ) if info1.get("type") == S3ObjectType.S3_OBJECT_TYPE_DIRECTORY: # Recursive copy() passes the directories too; see # S3FileSystem.cp_file. @@ -1224,7 +1269,10 @@ async def _copy_file(self, path1: str, path2: str, **kwargs) -> bool: ) finally: # See S3FileSystem._copy_file. - self._sync_fs.invalidate_cache(path2) + if isinstance(path2, str): + self._sync_fs.invalidate_cache(path2) + else: + self._sync_fs._invalidate(destination) return True async def _copy_object_with_multipart_upload( diff --git a/pyathena/filesystem/s3_path_pairing.py b/pyathena/filesystem/s3_path_pairing.py index 68e4a6a0a..dd12a4fda 100644 --- a/pyathena/filesystem/s3_path_pairing.py +++ b/pyathena/filesystem/s3_path_pairing.py @@ -27,10 +27,14 @@ class S3PathPairing: The sources are paired with their destinations as fsspec's ``copy()`` pairs them, except that a path with a version ID names that version of an object: it is not a glob pattern, and its destination is named after its - key without the version. A move compares the paths by what they name - (see :attr:`~pyathena.filesystem.s3_path.S3Path.target`), keeping ``null`` - versions distinct from their keys in the caller's versioning-enabled - buckets. + key without the version. A source may also be an + :class:`~pyathena.filesystem.s3_path.S3Path`, such as a key that a listing + returns, which names its bucket and key as they are, also when the key + ends in what looks like a version ID query, and which is never parsed + again (see :meth:`s3_copy_pairs`). A move compares the paths by what they + name (see :attr:`~pyathena.filesystem.s3_path.S3Path.target`), keeping + ``null`` versions distinct from their keys in the caller's + versioning-enabled buckets. :meth:`delete_paths` splits the paths of an ``rm()``. The pairing is a frozen dataclass of the paths as given, and holds no @@ -97,11 +101,15 @@ def looks_up_destination(self) -> bool: def copy_pairs( self, - sources: Sequence[str] | None = None, + sources: Sequence[str | S3Path] | None = None, destination_is_dir: bool | None = None, - ) -> list[tuple[str, str]]: + ) -> list[tuple[str | S3Path, str]]: """Pair the sources with their destinations as fsspec's ``copy()`` does. + A source string with a version ID is named after its key without the + version, and an ``S3Path`` source after its bucket and key (its + ``name``), whose suffix the destination keeps as it is. + Args: sources: The expansion of ``path1``, without the directories that :attr:`skips_directories` leaves out; needed when @@ -136,17 +144,64 @@ def copy_pairs( exists = source_is_str and ( (glob and len(sources) == 1) or (not glob and dest_is_dir and not trailing_sep(path1)) ) - names = [S3Path.split_version_id(p)[0] for p in sources] + names = [ + p.name if isinstance(p, S3Path) else S3Path.split_version_id(p)[0] for p in sources + ] destinations = other_paths(names, path2, exists=exists, flatten=not source_is_str) return list(zip(sources, destinations, strict=True)) + def s3_copy_pairs( + self, + sources: Sequence[str | S3Path] | None = None, + destination_is_dir: bool | None = None, + ) -> list[tuple[str | S3Path, str | S3Path]]: + """Pair the sources with their S3 destinations, as :meth:`copy_pairs` does. + + The destination of an ``S3Path`` source, when ``path2`` is a string, + is an ``S3Path`` too: ``path2`` parsed, with the suffix that + :meth:`copy_pairs` appends to it added to its key as it is, so that a + key such as ``a/x?versionId=null`` is copied to the key + ``b/x?versionId=null`` instead of being parsed as a version. The + other destinations are strings, as :meth:`copy_pairs` returns them. + + Args: + sources: As for :meth:`copy_pairs`. + destination_is_dir: As for :meth:`copy_pairs`. + + Returns: + The sources and their destinations. + + Raises: + ValueError: If ``sources`` or ``destination_is_dir`` is needed and + None, or if ``path2`` names a version, which cannot be + written, and a destination is an ``S3Path``. + """ + pairs = self.copy_pairs(sources, destination_is_dir) + if not isinstance(self.path2, str) or not any(isinstance(p1, S3Path) for p1, _ in pairs): + return list(pairs) + # Every destination that copy_pairs() names after a source starts with + # path2 without its trailing slashes, followed by the suffix. + root_name = self.path2.rstrip("/") + root = S3Path.parse(root_name) + if root.version_id: + raise ValueError("Cannot copy to a versioned file.") + typed: list[tuple[str | S3Path, str | S3Path]] = [] + for p1, p2 in pairs: + if isinstance(p1, S3Path): + suffix = p2[len(root_name) :] + key = f"{root.key}{suffix}" if root.key else suffix.removeprefix("/") + typed.append((p1, S3Path(root.bucket, key))) + else: + typed.append((p1, p2)) + return typed + def conflict_candidates( self, - pairs: Sequence[tuple[str, str]], + pairs: Sequence[tuple[str | S3Path, str | S3Path]], *, versioning_enabled_buckets: Collection[str] = (), copy: bool = False, - ) -> list[str]: + ) -> list[str | S3Path]: """Return the sources of a move whose conflicts depend on an object at their key. A source with another source below it may be a directory. If no object @@ -158,7 +213,9 @@ def conflict_candidates( Args: pairs: The sources and destinations of the move, as - :meth:`copy_pairs` of this pairing returns them. + :meth:`copy_pairs` or :meth:`s3_copy_pairs` of this pairing + returns them. A string is parsed; an ``S3Path`` is used as + it is. versioning_enabled_buckets: Buckets whose versioning is enabled. Their ``null`` versions are distinct from their unversioned keys. Use the same collection for :meth:`move_pairs`. @@ -167,32 +224,36 @@ def conflict_candidates( conflicts. Returns: - The sources, in ``bucket/key`` form (their - :attr:`~pyathena.filesystem.s3_path.S3Path.target`), that have - another source below them and a destination that conflicts, one - per pair in the order of the pairs; empty if nothing needs to be - looked up. + The sources that have another source below them and a + destination that conflicts, one per pair in the order of the + pairs, as their :attr:`~pyathena.filesystem.s3_path.S3Path.target`: + in ``bucket/key`` form for a source string, and as an ``S3Path`` + for an ``S3Path`` source. Empty if nothing needs to be looked up. Raises: TypeError: If ``versioning_enabled_buckets`` is a string. """ - return self._moves(pairs, versioning_enabled_buckets, copy)[2] + return [ + target if isinstance(p1, S3Path) else str(target) + for p1, target in self._moves(pairs, versioning_enabled_buckets, copy)[2] + ] def move_pairs( self, - pairs: Sequence[tuple[str, str]], - missing: Collection[str] | None = None, + pairs: Sequence[tuple[str | S3Path, str | S3Path]], + missing: Collection[str | S3Path] | None = None, *, versioning_enabled_buckets: Collection[str] = (), - ) -> list[tuple[str, str]]: + ) -> list[tuple[str | S3Path, str | S3Path]]: """Check the pairs of a move and leave out the sources that stay in place. Args: pairs: The sources and destinations of the move, as - :meth:`copy_pairs` of this pairing returns them. + :meth:`copy_pairs` or :meth:`s3_copy_pairs` of this pairing + returns them. missing: The :meth:`conflict_candidates` without an object at their - key, in any form that names them; needed when there are - candidates. + key, as strings in any form that names them or as ``S3Path`` + objects; needed when there are candidates. versioning_enabled_buckets: Buckets whose versioning is enabled. Their ``null`` versions are distinct from their unversioned keys. Use the same collection for :meth:`conflict_candidates`. @@ -211,12 +272,12 @@ def move_pairs( Also if ``versioning_enabled_buckets`` is a string. """ named = self._check(pairs, missing, versioning_enabled_buckets, "move") - return [(p1, p2) for p1, p2, _, source, dest in named if source != dest] + return [(pair.path1, pair.path2) for pair in named if pair.source != pair.destination] def check_copy_pairs( self, - pairs: Sequence[tuple[str, str]], - missing: Collection[str] | None = None, + pairs: Sequence[tuple[str | S3Path, str | S3Path]], + missing: Collection[str | S3Path] | None = None, *, versioning_enabled_buckets: Collection[str] = (), ) -> None: @@ -230,10 +291,11 @@ def check_copy_pairs( Args: pairs: The sources and destinations of the copy, as - :meth:`copy_pairs` of this pairing returns them. + :meth:`copy_pairs` or :meth:`s3_copy_pairs` of this pairing + returns them. missing: The :meth:`conflict_candidates` without an object at their - key, in any form that names them; needed when there are - candidates. + key, as strings in any form that names them or as ``S3Path`` + objects; needed when there are candidates. versioning_enabled_buckets: Buckets whose versioning is enabled. Use the same collection for :meth:`conflict_candidates`. @@ -249,11 +311,11 @@ def check_copy_pairs( def _check( self, - pairs: Sequence[tuple[str, str]], - missing: Collection[str] | None, + pairs: Sequence[tuple[str | S3Path, str | S3Path]], + missing: Collection[str | S3Path] | None, versioning_enabled_buckets: Collection[str], operation: str, - ) -> list[tuple[str, str, bool, str, str]]: + ) -> list[_NamedPair]: """Raise for the conflicts of a move or a copy. Args: @@ -279,15 +341,15 @@ def _check( if missing is None and candidates: raise ValueError("missing is needed to check the pairs.") # A directory without an object at its key writes no destination. - skipped = set(candidates).intersection( - self._target(S3Path.parse(path), versioning_enabled_buckets) for path in missing or () + skipped = {target for _, target in candidates}.intersection( + self._target(self._as_path(path), versioning_enabled_buckets) for path in missing or () ) # A path with a version always names an object, so it writes its # destination even when the key has no current object. writers = Counter( - dest - for _, _, versioned, source, dest in named - if source != dest and (versioned or source not in skipped) + pair.destination + for pair in named + if pair.source != pair.destination and (pair.versioned or pair.source not in skipped) ) for dest in writers: if writers[dest] > 1: @@ -337,7 +399,19 @@ def _is_glob(path: str) -> bool: return has_magic(path) and not S3Path.has_version_id(path) @staticmethod - def _target(path: S3Path, versioning_enabled_buckets: Collection[str]) -> str: + def _as_path(path: str | S3Path) -> S3Path: + """Return a path of the pairs as an ``S3Path``. + + Args: + path: A path string, which is parsed, or an ``S3Path``. + + Returns: + The path. + """ + return path if isinstance(path, S3Path) else S3Path.parse(path) + + @staticmethod + def _target(path: S3Path, versioning_enabled_buckets: Collection[str]) -> S3Path: """Return a move target using the caller's bucket versioning state. Args: @@ -345,17 +419,17 @@ def _target(path: S3Path, versioning_enabled_buckets: Collection[str]) -> str: versioning_enabled_buckets: Buckets whose versioning is enabled. Returns: - The normalized path, retaining a ``null`` version only when its - bucket has versioning enabled. + The path, retaining a ``null`` version only when its bucket has + versioning enabled. """ - return str(path if path.bucket in versioning_enabled_buckets else path.target) + return path if path.bucket in versioning_enabled_buckets else path.target @staticmethod def _moves( - pairs: Sequence[tuple[str, str]], + pairs: Sequence[tuple[str | S3Path, str | S3Path]], versioning_enabled_buckets: Collection[str] = (), copy: bool = False, - ) -> tuple[list[tuple[str, str, bool, str, str]], set[str], list[str]]: + ) -> tuple[list[_NamedPair], set[S3Path], list[tuple[str | S3Path, S3Path]]]: """Compare the paths of a move by what they name. Args: @@ -367,7 +441,8 @@ def _moves( Each pair with whether its source has a version and the targets of its source and destination; the targets of all sources, including those left in place; and the conflict candidates (see - :meth:`conflict_candidates`). + :meth:`conflict_candidates`), each as the source of its pair and + its target. Raises: TypeError: If ``versioning_enabled_buckets`` is a string. @@ -378,32 +453,52 @@ def _moves( ) named = [] for p1, p2 in pairs: - source_path = S3Path.parse(p1) + source_path = S3PathPairing._as_path(p1) named.append( - ( + _NamedPair( p1, p2, bool(source_path.version_id), S3PathPairing._target(source_path, versioning_enabled_buckets), - S3PathPairing._target(S3Path.parse(p2), versioning_enabled_buckets), + S3PathPairing._target(S3PathPairing._as_path(p2), versioning_enabled_buckets), ) ) # The sources left in place count too; a copy onto one overwrites it. - sources = {source for _, _, _, source, _ in named} - # A source with another source below it may be a directory. + sources = {pair.source for pair in named} + # A source with another source below it may be a directory, named + # as it is listed: the parents of a name are cut at its last slash. directories: set[str] = set() for source in sources: - parent = source.rpartition("/")[0] + parent = source.name.rpartition("/")[0] while parent and parent not in directories: directories.add(parent) parent = parent.rpartition("/")[0] - counts = Counter(dest for _, _, _, source, dest in named if source != dest) + counts = Counter(pair.destination for pair in named if pair.source != pair.destination) candidates = [ - source - for _, _, versioned, source, dest in named - if source != dest - and (counts[dest] > 1 or dest in sources) - and (copy or source in directories) - and not versioned + (pair.path1, pair.source) + for pair in named + if pair.source != pair.destination + and (counts[pair.destination] > 1 or pair.destination in sources) + and (copy or pair.source.name in directories) + and not pair.versioned ] return named, sources, candidates + + +@dataclass(frozen=True) +class _NamedPair: + """A pair of a move or a copy with the paths that it names. + + Attributes: + path1: The source, as given in the pairs. + path2: The destination, as given in the pairs. + versioned: Whether the source has a version. + source: The target of the source (see ``S3PathPairing._target``). + destination: The target of the destination. + """ + + path1: str | S3Path + path2: str | S3Path + versioned: bool + source: S3Path + destination: S3Path diff --git a/tests/pyathena/filesystem/test_s3.py b/tests/pyathena/filesystem/test_s3.py index ee9da2f9b..f8ea014e2 100644 --- a/tests/pyathena/filesystem/test_s3.py +++ b/tests/pyathena/filesystem/test_s3.py @@ -68,6 +68,7 @@ MULTIPART_COPY_BLOCK_SIZE, MULTIPART_COPY_KWARGS, MULTIPART_COPY_SIZE, + BranchRecordingCallback, OpaquePath, ThrottledOperation, forking, @@ -77,6 +78,7 @@ poll_until, retried, run_in_forked_child, + s3_copy_targets, s3_request_targets, serve_s3_keys, shutdown_bounded, @@ -1256,6 +1258,67 @@ def test_fan_outs_act_on_listed_keys(self, call, result, sent): assert call(fs) == result assert s3_request_targets(requests) == sent + # The key "a/x" and a key that a string path would name as its null + # version, which a move compares with the key itself. + NULL_KEYS = ("a/x", "a/x?versionId=null") + + @pytest.mark.parametrize( + ("call", "stored", "copied", "deleted"), + [ + pytest.param( + lambda fs: fs.copy("s3://bucket/a", "s3://bucket/b", recursive=True), + {"a/x", "a/x?versionId=null", "b/x", "b/x?versionId=null"}, + [("a/x", None, "b/x"), ("a/x?versionId=null", None, "b/x?versionId=null")], + [], + id="copy", + ), + pytest.param( + lambda fs: fs.copy("s3://bucket/a/*", "s3://bucket/c/"), + {"a/x", "a/x?versionId=null", "c/x", "c/x?versionId=null"}, + [("a/x", None, "c/x"), ("a/x?versionId=null", None, "c/x?versionId=null")], + [], + id="copy-glob", + ), + pytest.param( + lambda fs: fs.mv("s3://bucket/a", "s3://bucket/b", recursive=True), + {"b/x", "b/x?versionId=null"}, + [("a/x", None, "b/x"), ("a/x?versionId=null", None, "b/x?versionId=null")], + [("delete_objects", "a/x", None), ("delete_objects", "a/x?versionId=null", None)], + id="mv", + ), + ], + ) + def test_copies_act_on_listed_keys(self, call, stored, copied, deleted): + # GH-1403: copy() and mv() copy each key that a listing returns to a + # destination named after the key as it is, also a key that ends in + # what looks like a version ID query, never the version that parsing + # its name would give. + fs = S3FileSystem( + key="dummy", secret="dummy", region_name="us-east-1", skip_instance_cache=True + ) + store, requests = serve_s3_keys(fs, self.NULL_KEYS) + call(fs) + assert store == stored + assert s3_copy_targets(requests) == copied + assert [t for t in s3_request_targets(requests) if t[0] == "delete_objects"] == deleted + + def test_get_downloads_listed_keys(self, tmp_path): + # GH-1403: get() downloads each key that a listing returns by its + # bucket and key to a local path named after the key as it is, and + # names the branch of the callback after the key as a string. + fs = S3FileSystem( + key="dummy", secret="dummy", region_name="us-east-1", skip_instance_cache=True + ) + _, requests = serve_s3_keys(fs, self.NULL_KEYS) + callback = BranchRecordingCallback() + fs.get("s3://bucket/a", str(tmp_path / "out"), recursive=True, callback=callback) + assert sorted(os.listdir(tmp_path / "out")) == ["x", "x?versionId=null"] + assert [t for t in s3_request_targets(requests) if t[0] == "get_object"] == [ + ("get_object", "a/x", None), + ("get_object", "a/x?versionId=null", None), + ] + assert sorted(callback.sources) == ["bucket/a", "bucket/a/x", "bucket/a/x?versionId=null"] + def test_expand_path_expands_below_listed_matches(self): # GH-1402: a recursive glob expands below each match by its key prefix. fs = S3FileSystem( @@ -2231,7 +2294,9 @@ def test_mv_conflicting_destinations(self, path1, path2): def test_mv_conflicting_destinations_of_listed_sources(self): # A list of sources is mapped without looking up the destination. fs = self._make_fs() - fs.expand_path = mock.MagicMock(return_value=["bucket/x/a", "bucket/y/a"]) + fs._expand_paths = mock.MagicMock( + return_value=[(p, S3Path.parse(p)) for p in ("bucket/x/a", "bucket/y/a")] + ) fs.isdir = mock.MagicMock() with pytest.raises(ValueError, match="same destination"): @@ -2707,8 +2772,10 @@ def test_freed_by_reference_counting(self): def test_mv_nothing_within_maxdepth(self): # Only directories within maxdepth: nothing is moved, as with copy(). fs = self._make_fs() - fs.expand_path = mock.MagicMock(return_value=["bucket/src/sub"]) - fs.isdir = mock.MagicMock(return_value=True) + fs._expand_paths = mock.MagicMock( + return_value=[("bucket/src/sub", S3Path("bucket", "src/sub"))] + ) + fs._target_isdir = mock.MagicMock(return_value=True) fs.mv("s3://bucket/src", "s3://bucket/dst/", recursive=True, maxdepth=1) fs._call.assert_not_called() @@ -3063,11 +3130,12 @@ def test_get_put_pass_kwargs_to_expansion(self, tmp_path): ) as expand_path: fs.put(str(tmp_path / "a"), "s3://bucket/dst/", assume_literal=True) assert expand_path.call_args.kwargs["assume_literal"] is True - fs.expand_path = mock.MagicMock(return_value=["bucket/root/a"]) + fs._expand_paths = mock.MagicMock( + return_value=[("bucket/root/a", S3Path("bucket", "root/a"))] + ) fs.get_file = mock.MagicMock() - fs.isdir = mock.MagicMock(return_value=False) fs.get("s3://bucket/root/", str(tmp_path / "out") + "/", recursive=True, prefix="a") - assert fs.expand_path.call_args.kwargs["prefix"] == "a" + assert fs._expand_paths.call_args.kwargs["prefix"] == "a" @pytest.mark.parametrize("on_error", ["return", "raise"]) def test_cat_ranges_on_error(self, on_error): diff --git a/tests/pyathena/filesystem/test_s3_async.py b/tests/pyathena/filesystem/test_s3_async.py index 23574a21b..fc733ed76 100644 --- a/tests/pyathena/filesystem/test_s3_async.py +++ b/tests/pyathena/filesystem/test_s3_async.py @@ -50,6 +50,7 @@ MOVE_SOURCE_PARAMS, MULTIPART_COPY_BLOCK_SIZE, MULTIPART_COPY_KWARGS, + BranchRecordingCallback, OpaquePath, ThrottledOperation, await_bounded, @@ -62,6 +63,7 @@ retried, run_bounded, run_in_forked_child, + s3_copy_targets, s3_request_targets, serve_s3_keys, shutdown_bounded, @@ -736,6 +738,64 @@ async def test_fan_outs_act_on_listed_keys(self, call, result, sent): assert await await_bounded(call(fs)) == result assert s3_request_targets(requests) == sent + @pytest.mark.parametrize( + ("call", "stored", "copied", "deleted"), + [ + pytest.param( + lambda fs: fs._copy("s3://bucket/a", "s3://bucket/b", recursive=True), + {"a/x", "a/x?versionId=null", "b/x", "b/x?versionId=null"}, + [("a/x", None, "b/x"), ("a/x?versionId=null", None, "b/x?versionId=null")], + [], + id="copy", + ), + pytest.param( + lambda fs: fs._copy("s3://bucket/a/*", "s3://bucket/c/"), + {"a/x", "a/x?versionId=null", "c/x", "c/x?versionId=null"}, + [("a/x", None, "c/x"), ("a/x?versionId=null", None, "c/x?versionId=null")], + [], + id="copy-glob", + ), + pytest.param( + lambda fs: fs._mv("s3://bucket/a", "s3://bucket/b", recursive=True), + {"b/x", "b/x?versionId=null"}, + [("a/x", None, "b/x"), ("a/x?versionId=null", None, "b/x?versionId=null")], + [("delete_objects", "a/x", None), ("delete_objects", "a/x?versionId=null", None)], + id="mv", + ), + ], + ) + @pytest.mark.asyncio + async def test_copies_act_on_listed_keys(self, call, stored, copied, deleted): + # GH-1403: the asynchronous copies act on the keys that a listing + # returns, as those of S3FileSystem do. + fs = AioS3FileSystem( + key="dummy", secret="dummy", region_name="us-east-1", skip_instance_cache=True + ) + store, requests = serve_s3_keys(fs._sync_fs, {"a/x", "a/x?versionId=null"}) + await await_bounded(call(fs)) + assert store == stored + assert s3_copy_targets(requests) == copied + assert [t for t in s3_request_targets(requests) if t[0] == "delete_objects"] == deleted + + @pytest.mark.asyncio + async def test_get_downloads_listed_keys(self, tmp_path): + # GH-1403: _get() downloads the keys that a listing returns as get() + # does, with the branches of the callback named after them as strings. + fs = AioS3FileSystem( + key="dummy", secret="dummy", region_name="us-east-1", skip_instance_cache=True + ) + _, requests = serve_s3_keys(fs._sync_fs, {"a/x", "a/x?versionId=null"}) + callback = BranchRecordingCallback() + await await_bounded( + fs._get("s3://bucket/a", str(tmp_path / "out"), recursive=True, callback=callback) + ) + assert sorted(os.listdir(tmp_path / "out")) == ["x", "x?versionId=null"] + assert [t for t in s3_request_targets(requests) if t[0] == "get_object"] == [ + ("get_object", "a/x", None), + ("get_object", "a/x?versionId=null", None), + ] + assert sorted(callback.sources) == ["bucket/a", "bucket/a/x", "bucket/a/x?versionId=null"] + @pytest.mark.asyncio async def test_du_cancellation_sends_no_further_lookup(self): # GH-1402: as fsspec's _du(), the lookups of the found entries are diff --git a/tests/pyathena/filesystem/test_s3_path_pairing.py b/tests/pyathena/filesystem/test_s3_path_pairing.py index 6685a4dc2..933758c7e 100644 --- a/tests/pyathena/filesystem/test_s3_path_pairing.py +++ b/tests/pyathena/filesystem/test_s3_path_pairing.py @@ -7,6 +7,7 @@ import pytest +from pyathena.filesystem.s3_path import S3Path from pyathena.filesystem.s3_path_pairing import S3PathPairing @@ -125,6 +126,69 @@ def test_looks_up_destination(self, path1, path2, expected): def test_copy_pairs(self, path1, path2, sources, destination_is_dir, expected): assert S3PathPairing(path1, path2).copy_pairs(sources, destination_is_dir) == expected + def test_copy_pairs_names_path_sources_by_key(self): + # GH-1403: an S3Path source, such as a listed key, is named after its + # bucket and key as they are, while a source string with a version ID + # is named after its key. + listed = S3Path("bucket", "d/x?versionId=null") + pairing = S3PathPairing("s3://bucket/d", "out", recursive=True) + assert pairing.copy_pairs(["bucket/d", listed], False) == [ + ("bucket/d", "out"), + (listed, "out/x?versionId=null"), + ] + + @pytest.mark.parametrize( + ("path1", "path2", "sources", "destination_is_dir", "expected"), + [ + # The destination root parsed, with the suffix of the listed key. + ( + "s3://bucket/d", + "s3://bucket/out", + ["bucket/d", S3Path("bucket", "d/x?versionId=null")], + False, + [ + ("bucket/d", "s3://bucket/out"), + ( + S3Path("bucket", "d/x?versionId=null"), + S3Path("bucket", "out/x?versionId=null"), + ), + ], + ), + # A bucket root, given with a trailing slash. + ( + "s3://bucket/d/*", + "s3://other/", + [S3Path("bucket", "d/x?versionId=null")], + None, + [(S3Path("bucket", "d/x?versionId=null"), S3Path("other", "x?versionId=null"))], + ), + # A list of sources, copied into the destination by name. + ( + ["s3://bucket/d"], + "s3://bucket/out", + ["bucket/d", S3Path("bucket", "d/x?versionId=null")], + None, + [ + ("bucket/d", "s3://bucket/out/d"), + ( + S3Path("bucket", "d/x?versionId=null"), + S3Path("bucket", "out/x?versionId=null"), + ), + ], + ), + ], + ) + def test_s3_copy_pairs(self, path1, path2, sources, destination_is_dir, expected): + # GH-1403: the destination of an S3Path source is an S3Path, never a + # string that parsing would read as a version. + pairing = S3PathPairing(path1, path2, recursive=True) + assert pairing.s3_copy_pairs(sources, destination_is_dir) == expected + + def test_s3_copy_pairs_versioned_destination(self): + pairing = S3PathPairing("s3://bucket/d", "s3://bucket/out?versionId=v1", recursive=True) + with pytest.raises(ValueError, match="versioned"): + pairing.s3_copy_pairs([S3Path("bucket", "d/x")], False) + def test_copy_pairs_needs_destination_lookup(self): with pytest.raises(ValueError, match="destination_is_dir"): S3PathPairing("s3://bucket/a", "s3://bucket/b").copy_pairs(["bucket/a"]) @@ -214,6 +278,28 @@ def test_move_pairs_directory_without_object(self): # The missing sources can be given in any form that names them. assert _pairing(pairs).move_pairs(pairs, missing={"s3://bucket/d"}) == pairs + def test_move_pairs_keeps_listed_keys_apart_from_versions(self): + # GH-1403: the S3Path of a key that ends in a null version query is + # compared as that key, not as the null version, which is the key + # without the query. + pairs = [ + (S3Path("bucket", "a/x"), S3Path("bucket", "b/x")), + (S3Path("bucket", "a/x?versionId=null"), S3Path("bucket", "b/x?versionId=null")), + ] + assert _pairing(pairs).move_pairs(pairs) == pairs + + def test_move_pairs_path_conflict_candidates(self): + # GH-1403: the candidates of S3Path sources are S3Path objects, and + # missing ones may be given as such; their directories are named as + # they are listed. + directory = S3Path("bucket", "d?versionId=v1") + pairs = [ + (directory, S3Path("bucket", "e")), + (S3Path("bucket", "d?versionId=v1/x"), S3Path("bucket", "e")), + ] + assert _pairing(pairs).conflict_candidates(pairs) == [directory] + assert _pairing(pairs).move_pairs(pairs, missing={directory}) == pairs + def test_move_pairs_missing_is_a_collection(self): pairs = [("s3://bucket/d", "s3://bucket/e"), ("s3://bucket/d/x", "s3://bucket/e")] with pytest.raises(TypeError, match="collection"): diff --git a/tests/pyathena/util.py b/tests/pyathena/util.py index d39594b73..18e9de745 100644 --- a/tests/pyathena/util.py +++ b/tests/pyathena/util.py @@ -29,6 +29,7 @@ from botocore.config import Config from botocore.exceptions import ClientError from botocore.response import StreamingBody +from fsspec.callbacks import Callback from fsspec.implementations.dirfs import DirFileSystem from fsspec.implementations.local import LocalFileSystem from jinja2 import Environment, FileSystemLoader @@ -1812,6 +1813,39 @@ def __fspath__(self): return self._path +class BranchRecordingCallback(Callback): + """A progress callback that records the source paths of its branches.""" + + def __init__(self): + super().__init__() + self.sources = [] + + def branched(self, path_1, path_2, **kwargs): + self.sources.append(path_1) + return super().branched(path_1, path_2, **kwargs) + + +def s3_copy_targets(requests): + """Return the sources and destinations of recorded CopyObject requests. + + Args: + requests: The requests that :func:`serve_s3_keys` records. + + Returns: + The sorted tuples of the source key, the source version ID, and the + destination key. + """ + return sorted( + ( + request["CopySource"]["Key"], + request["CopySource"].get("VersionId"), + request["Key"], + ) + for name, request in requests + if name == "copy_object" + ) + + def s3_request_targets(requests): """Return the operations, keys and version IDs of recorded S3 requests. From fa1d263e58004cc72647564f01b8724dcfb1c83a Mon Sep 17 00:00:00 2001 From: laughingman7743 Date: Tue, 17 Nov 2026 03:35:18 +0900 Subject: [PATCH 2/2] Look up typed paths through info() and expand each object once (GH-1403) Review repairs: - The expansion names an object once: a path that the caller passes and a listed key that name the same object are returned once, as the first of them, as expand_path() names them once. copy(), mv() and get() raised that two paths had the same destination for, e.g., ["s3://bucket/a/x", "s3://bucket/a/*"], and rm() sent the key twice. - info() and cat_file(), and the async _info() and _cat_file(), accept an S3Path, which they look up and read by its bucket, key and version, so that the copies, the downloads, the files and the fan-outs call them for both forms instead of choosing a typed helper at each call; _target_info(), _target_isdir(), _cat_target() and the async _cat_object() are removed. - S3Path.of() returns a path string parsed or an S3Path as it is, in place of a helper on the filesystem and one on the pairing. - cp_file() and _cp_file() keep an S3Path from stringify_path(), a copy invalidates its destination by its bucket and key, and the conflict candidates of the pairing are its named pairs. - The async _copy() pairs lists without a thread, passes fsspec no maxdepth that it would not use, and says that its pairs are not checked for conflicts (GH-1398); _get_pairs() returns nothing. Co-Authored-By: Claude Opus 5.5 --- pyathena/filesystem/s3.py | 228 ++++++++-------------- pyathena/filesystem/s3_async.py | 88 +++------ pyathena/filesystem/s3_path.py | 19 ++ pyathena/filesystem/s3_path_pairing.py | 33 +--- tests/pyathena/filesystem/test_s3.py | 52 ++++- tests/pyathena/filesystem/test_s3_path.py | 7 + 6 files changed, 194 insertions(+), 233 deletions(-) diff --git a/pyathena/filesystem/s3.py b/pyathena/filesystem/s3.py index ebaa4798b..0acd0515e 100644 --- a/pyathena/filesystem/s3.py +++ b/pyathena/filesystem/s3.py @@ -1162,7 +1162,7 @@ def _ls_directory( raise FileNotFoundError(directory.name) return list(files) - def info(self, path: str | os.PathLike[str], **kwargs) -> S3Object: + def info(self, path: str | os.PathLike[str] | S3Path, **kwargs) -> S3Object: """Return information about an S3 path. Uses the directory cache first: a cached entry for the path is @@ -1192,7 +1192,9 @@ def info(self, path: str | os.PathLike[str], **kwargs) -> S3Object: parameters are used. Args: - path: S3 path (e.g., "s3://bucket" or "s3://bucket/key"). + path: S3 path (e.g., "s3://bucket" or "s3://bucket/key"), or an + ``S3Path``, whose bucket, key and version are looked up as + they are, such as a key that a listing returns. **kwargs: Additional arguments including: refresh: If True, bypass the cache and query S3. version_id: The version ID to look up when the path has none. @@ -1208,24 +1210,27 @@ def info(self, path: str | os.PathLike[str], **kwargs) -> S3Object: FileNotFoundError: If the path does not exist. """ refresh = kwargs.pop("refresh", False) - path = self._strip_protocol(path) - if path in ["/", ""]: - # S3Path.parse rejects the root path. - return S3Object( - init={ - "ContentLength": 0, - "ContentType": None, - "StorageClass": S3StorageClass.S3_STORAGE_CLASS_BUCKET, - "ETag": None, - "LastModified": None, - }, - type=S3ObjectType.S3_OBJECT_TYPE_DIRECTORY, - bucket="", - key=None, - version_id=None, - ) - s3_path = S3Path.parse(path) version_id = kwargs.pop("version_id", None) + if isinstance(path, S3Path): + s3_path = path + else: + name = self._strip_protocol(path) + if name in ["/", ""]: + # S3Path.parse rejects the root path. + return S3Object( + init={ + "ContentLength": 0, + "ContentType": None, + "StorageClass": S3StorageClass.S3_STORAGE_CLASS_BUCKET, + "ETag": None, + "LastModified": None, + }, + type=S3ObjectType.S3_OBJECT_TYPE_DIRECTORY, + bucket="", + key=None, + version_id=None, + ) + s3_path = S3Path.parse(name) if version_id and not s3_path.version_id: s3_path = s3_path.with_version_id(version_id) return self._path_info( @@ -1775,25 +1780,10 @@ def du(self, path, total=True, maxdepth=None, withdirs=False, **kwargs): info = self.info(path) sizes[info["name"]] = info["size"] for _, target in self._find_paths(path, maxdepth=maxdepth, withdirs=withdirs, **kwargs): - info = self._target_info(target) + info = self.info(target) sizes[info["name"]] = info["size"] return sum(sizes.values()) if total else sizes - def _target_info(self, target: str | S3Path) -> S3Object: - """Look up a path that an expansion or ``find()`` names. - - Args: - target: A path string that the caller passed, which ``info()`` - looks up, or a listed path, looked up by its bucket and key. - - Returns: - S3Object describing the bucket, directory, or file. - - Raises: - FileNotFoundError: If the path does not exist. - """ - return self.info(target) if isinstance(target, str) else self._path_info(target) - def glob(self, path, maxdepth=None, **kwargs): """Find the paths that match a glob pattern. @@ -1966,7 +1956,9 @@ def _expand_paths( The names, as :meth:`expand_path` returns them, and the paths that they name, sorted by name: the name itself for a path that the caller passed, which the caller parses as a path string, and - an ``S3Path`` for a listed entry. + an ``S3Path`` for a listed entry. A path that the caller passed + and a listed entry that name the same object are returned once, + as the first of them. Raises: ValueError: If ``maxdepth`` is less than 1. @@ -1987,7 +1979,16 @@ def _expand_paths( raise if not out: raise FileNotFoundError(path) - return sorted(out, key=lambda item: item[0]) + # The first name and path of each object, by what the path names. + expanded: dict[S3Path | str, tuple[str, str | S3Path]] = {} + for name, target in out: + try: + identity: S3Path | str = S3Path.of(target) + except ValueError: + # Such as the root path, which names no bucket. + identity = target + expanded.setdefault(identity, (name, target)) + return sorted(expanded.values(), key=lambda item: item[0]) def _expand_unversioned( self, paths: list[str], recursive: bool, maxdepth: int | None, **kwargs @@ -2831,7 +2832,7 @@ def _conflict_lookups( Raises: OSError: If bucket versioning cannot be read. """ - paths = {self._pair_path(p) for pair in pairs for p in pair} + paths = {S3Path.of(p) for pair in pairs for p in pair} unversioned = {p.name for p in paths if not p.version_id} buckets = { p.bucket @@ -2847,23 +2848,10 @@ def _conflict_lookups( for source in pairing.conflict_candidates( pairs, versioning_enabled_buckets=versioned_buckets, copy=copy ) - if self._head_object(self._pair_path(source), lookup_kwargs=lookup_kwargs) is None + if self._head_object(S3Path.of(source), lookup_kwargs=lookup_kwargs) is None } return missing, versioned_buckets - @staticmethod - def _pair_path(path: str | S3Path) -> S3Path: - """Return a path of the pairs of a copy as an ``S3Path``. - - Args: - path: A path string that the caller passed, which is parsed, or - a listed path. - - Returns: - The path. - """ - return path if isinstance(path, S3Path) else S3Path.parse(path) - def _copy_pairs( self, pairing: S3PathPairing, @@ -2890,16 +2878,13 @@ def _copy_pairs( """ if not pairing.expands: return list(pairing.copy_pairs()) - expanded = self._expand_paths( - pairing.path1, recursive=pairing.recursive, maxdepth=pairing.maxdepth, **kwargs - ) - if pairing.skips_directories: - expanded = [ - (name, target) - for name, target in expanded - if not (trailing_sep(name) or self._target_isdir(target)) - ] - sources = [target for _, target in expanded] + sources = [ + target + for name, target in self._expand_paths( + pairing.path1, recursive=pairing.recursive, maxdepth=pairing.maxdepth, **kwargs + ) + if not (pairing.skips_directories and (trailing_sep(name) or self.isdir(target))) + ] isdir = LocalFileSystem().isdir if local else self.isdir destination_is_dir = ( # A string, as looks_up_destination checks. @@ -2909,25 +2894,6 @@ def _copy_pairs( return list(pairing.copy_pairs(sources, destination_is_dir)) return pairing.s3_copy_pairs(sources, destination_is_dir) - def _target_isdir(self, target: str | S3Path) -> bool: - """Return whether a path that an expansion names is a directory. - - Args: - target: A path string that the caller passed, which ``isdir()`` - looks up, or a listed path, looked up by its bucket and key. - - Returns: - Whether the path is a directory. - """ - if isinstance(target, str): - return bool(self.isdir(target)) - try: - return bool( - self._path_info(target).get("type") == S3ObjectType.S3_OBJECT_TYPE_DIRECTORY - ) - except OSError: - return False - def copy( self, path1, @@ -3167,8 +3133,11 @@ def cp_file( A directory ``path1``, which recursive ``copy()`` passes along with the files under it, is skipped. """ - path1, path2 = stringify_path(path1), stringify_path(path2) - self._copy_file(path1, path2, **kwargs) + self._copy_file( + path1 if isinstance(path1, S3Path) else stringify_path(path1), + path2 if isinstance(path2, S3Path) else stringify_path(path2), + **kwargs, + ) def _copy_file(self, path1: str | S3Path, path2: str | S3Path, **kwargs) -> bool: """Copy an S3 object as :meth:`cp_file` does. @@ -3188,8 +3157,8 @@ def _copy_file(self, path1: str | S3Path, path2: str | S3Path, **kwargs) -> bool # Parameters of the multipart copy, not of the S3 requests. block_size = kwargs.pop("block_size", None) max_workers = kwargs.pop("max_workers", None) - source = self._pair_path(path1) - destination = self._pair_path(path2) + source = S3Path.of(path1) + destination = S3Path.of(path2) if destination.version_id: raise ValueError("Cannot copy to a versioned file.") if not source.key or not destination.key: @@ -3197,12 +3166,7 @@ def _copy_file(self, path1: str | S3Path, path2: str | S3Path, **kwargs) -> bool # Looked up with the parameters that authorize the access to the # source, such as its customer-provided key. - source_params = self.core.copy_source_params(kwargs) - info1 = ( - self.info(path1, **source_params) - if isinstance(path1, str) - else self._path_info(source, lookup_kwargs=self._get_lookup_kwargs(source_params)) - ) + info1 = self.info(path1, **self.core.copy_source_params(kwargs)) if info1.get("type") == S3ObjectType.S3_OBJECT_TYPE_DIRECTORY: # Recursive copy() passes the directories too; S3 has no object # to copy for them. @@ -3223,10 +3187,7 @@ def _copy_file(self, path1: str | S3Path, path2: str | S3Path, **kwargs) -> bool finally: # A multipart copy that fails to copy an annotation has already # written the destination. - if isinstance(path2, str): - self.invalidate_cache(path2) - else: - self._invalidate(destination) + self._invalidate(destination) return True def _copy_object_with_multipart_upload( @@ -3495,13 +3456,13 @@ def cat(self, path, recursive=False, on_error="raise", batch_size=None, **kwargs targets = self._first_targets(self._expand_paths(path, recursive=recursive, **kwargs)) paths = list(targets) if not (len(paths) > 1 or isinstance(path, list) or paths[0] != self._strip_protocol(path)): - return self._cat_target(targets[paths[0]], **kwargs) + return self.cat_file(targets[paths[0]], **kwargs) outcomes: list[Any] | None = None contents: dict[str, Any] | None = None outcome = None try: outcomes = self._run_batch( - functools.partial(self._cat_target, **kwargs), + functools.partial(self.cat_file, **kwargs), list(targets.values()), min(limit, len(paths)), counts=None if on_error == "raise" else lambda e: False, @@ -3538,22 +3499,6 @@ def _first_targets(expanded: list[tuple[str, str | S3Path]]) -> dict[str, str | targets.setdefault(name, target) return targets - def _cat_target(self, target: str | S3Path, **kwargs) -> bytes: - """Read a path of an expansion. - - Args: - target: A path string that the caller passed, which - :meth:`cat_file` reads, or a listed path, read by its bucket - and key. - **kwargs: Additional parameters, as for :meth:`cat_file`. - - Returns: - The bytes read from the object. - """ - if isinstance(target, str): - return self.cat_file(target, **kwargs) - return self._cat_object(target, **kwargs) - def cat_ranges( self, paths, starts, ends, max_gap=None, on_error="return", batch_size=None, **kwargs ) -> list[bytes | Exception]: @@ -3621,7 +3566,7 @@ def cat_range(item: tuple[str, int | None, int | None]) -> bytes: def cat_file( self, - path: str | os.PathLike[str], + path: str | os.PathLike[str] | S3Path, start: int | None = None, end: int | None = None, **kwargs, @@ -3638,7 +3583,9 @@ def cat_file( range and receives the lookup parameters among ``kwargs``. Args: - path: S3 path (s3://bucket/key) of the object. + path: S3 path (s3://bucket/key) of the object, or an ``S3Path``, + whose bucket, key and version are read as they are, such as a + key that a listing returns. start: Byte offset to start reading at. A negative value counts from the end of the object. end: Byte offset to stop reading at (exclusive). A negative value @@ -3654,6 +3601,8 @@ def cat_file( FileNotFoundError: If the path has no key or the key does not exist. """ + if isinstance(path, S3Path): + return self._cat_object(path, start=start, end=end, **kwargs) path = stringify_path(path) return self._cat_object(S3Path.parse(path), start=start, end=end, name=path, **kwargs) @@ -3704,12 +3653,8 @@ def _cat_object( # empty range sends no GetObject request that would report a # missing object. lookup_kwargs = self._get_lookup_kwargs(kwargs) - info = ( - self.info(name, version_id=version_id, **lookup_kwargs) - if name is not None - else self._path_info( - path.with_version_id(version_id), lookup_kwargs=lookup_kwargs - ) + info = self.info( + name if name is not None else path, version_id=version_id, **lookup_kwargs ) if ( info.get("type") == S3ObjectType.S3_OBJECT_TYPE_DIRECTORY @@ -3954,13 +3899,13 @@ def get_file( ``DEFAULT_BLOCK_SIZE`` if that is not positive either, and the size of the callback is unknown. """ - s3_path = rpath if isinstance(rpath, S3Path) else S3Path.parse(self._strip_protocol(rpath)) + s3_path = S3Path.of(rpath if isinstance(rpath, S3Path) else self._strip_protocol(rpath)) if outfile is None and isfilelike(lpath): outfile = lpath elif ( outfile is None and not (s3_path.version_id or kwargs.get("version_id")) - and (self.isdir(rpath) if not isinstance(rpath, S3Path) else self._target_isdir(rpath)) + and self.isdir(rpath) ): # A requested version always names an object, while isdir() # would look up the latest version, or the prefix of the same @@ -4545,7 +4490,7 @@ def _key_exists(self, path: S3Path, lookup_kwargs: Mapping[str, Any]) -> bool: True if an object exists at the key. """ if not (path.key or "").endswith("/"): - return self._object_exists(path.uri, lookup_kwargs) + return self._object_exists(path, lookup_kwargs) try: self.core.head_object(path, **lookup_kwargs) except FileNotFoundError: @@ -4567,11 +4512,7 @@ def _object_exists(self, path: str | S3Path, lookup_kwargs: Mapping[str, Any]) - True if an object exists at the path. """ try: - info = ( - self.info(path, **lookup_kwargs) - if isinstance(path, str) - else self._path_info(path, lookup_kwargs=lookup_kwargs) - ) + info = self.info(path, **lookup_kwargs) except FileNotFoundError: return False return bool(info.get("type") != S3ObjectType.S3_OBJECT_TYPE_DIRECTORY) @@ -4880,10 +4821,10 @@ def resolve( """ # A new dictionary, so that the caller's is not modified. kwargs: dict[str, Any] = dict(request_kwargs or {}) - # A path string is looked up as a string, and an S3Path by its bucket, - # key and version. - typed = path if isinstance(path, S3Path) else None - s3_path = typed or S3Path.parse(cast(str, path)) + # The path as it is looked up: a path string as a string, and an + # S3Path by its bucket, key and version. + target = path + s3_path = S3Path.of(path) path = str(path) path_version_id = s3_path.version_id if not s3_path.key: @@ -4902,6 +4843,7 @@ def resolve( # Carry the version in the path, as with the ?versionId= suffix, # so that a reopened (e.g., unpickled) file reads the same version. path = f"{path}?versionId={version_id}" + target = s3_path.with_version_id(version_id) if isinstance(target, S3Path) else path if "r" not in mode and not ( fs.core.MULTIPART_UPLOAD_MIN_PART_SIZE <= block_size @@ -4924,11 +4866,7 @@ def resolve( if "r" in mode: # Looked up here; the base class initializer of a file would # otherwise take the size from the latest version of the object. - info = ( - fs._path_info(typed.with_version_id(version_id), lookup_kwargs=lookup_kwargs) - if typed - else fs.info(path, version_id=version_id, **lookup_kwargs) - ) + info = fs.info(target, version_id=version_id, **lookup_kwargs) if info.get("type") == S3ObjectType.S3_OBJECT_TYPE_DIRECTORY: # A prefix has no object to read. raise FileNotFoundError(path) @@ -4952,11 +4890,7 @@ def resolve( # The rewritten object keeps the metadata of the existing one, # which a cached listing entry lacks, so look up the object. with contextlib.suppress(FileNotFoundError): - appended = ( - fs._path_info(typed, refresh=True, lookup_kwargs=lookup_kwargs) - if typed - else fs.info(path, refresh=True, **lookup_kwargs) - ) + appended = fs.info(target, refresh=True, **lookup_kwargs) if ( appended is not None and appended.get("type") == S3ObjectType.S3_OBJECT_TYPE_DIRECTORY @@ -4972,11 +4906,7 @@ def resolve( # the whole object is read, and an object replaced since # is not. condition = {"IfMatch": etag} if etag else {} - append_data = ( - fs._cat_object(typed, **condition, **lookup_kwargs) - if typed - else fs.cat_file(path, **condition, **lookup_kwargs) - ) + append_data = fs.cat_file(target, **condition, **lookup_kwargs) # The attributes of the object looked up are kept where the # caller gives none: the encryption as a whole, so that its # fields are not combined with another encryption, and @@ -4999,7 +4929,7 @@ def resolve( elif "x" in mode: # Checked up front so that no data is uploaded for an existing # object, and on commit with IfNoneMatch for one created since. - if fs._object_exists(typed or path, lookup_kwargs): + if fs._object_exists(target, lookup_kwargs): raise FileExistsError(path) kwargs.update({"IfNoneMatch": "*"}) return cls( diff --git a/pyathena/filesystem/s3_async.py b/pyathena/filesystem/s3_async.py index 30f19331a..2cc9a7931 100644 --- a/pyathena/filesystem/s3_async.py +++ b/pyathena/filesystem/s3_async.py @@ -435,7 +435,7 @@ async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: """Release the threads with :meth:`close`.""" self.close() - async def _info(self, path: str | os.PathLike[str], **kwargs) -> S3Object: + async def _info(self, path: str | os.PathLike[str] | S3Path, **kwargs) -> S3Object: return await self._thread_pool.run(self._sync_fs.info, path, **kwargs) async def _ls( @@ -445,7 +445,7 @@ async def _ls( async def _cat_file( self, - path: str | os.PathLike[str], + path: str | os.PathLike[str] | S3Path, start: int | None = None, end: int | None = None, **kwargs, @@ -824,11 +824,12 @@ async def _copy( ) -> None: """Copy files within S3. - See :meth:`S3FileSystem.copy`. The paths are paired as there, and the - copies run as in fsspec's ``_copy()``, ``batch_size`` of them at once - as tasks. A cancellation cancels the copies and waits for them; a - copy that is already being cancelled, such as by the shutdown of the - event loop, is not cancelled again. + See :meth:`S3FileSystem.copy`. The paths are paired as there, but the + pairs are not checked for conflicts, and the copies run as in + fsspec's ``_copy()``, ``batch_size`` of them at once as tasks. A + cancellation cancels the copies and waits for them; a copy that is + already being cancelled, such as by the shutdown of the event loop, + is not cancelled again. Args: path1: Source S3 path, glob pattern, or list of them. @@ -844,9 +845,11 @@ async def _copy( # Paired as the sync copy() pairs them, so that the keys that a # listing returns are copied by their buckets and keys; fsspec copies # the lists of the pairs. - pairs = await self._thread_pool.run( - self._sync_fs._copy_pairs, - S3PathPairing(path1, path2, recursive=recursive, maxdepth=maxdepth), + pairing = S3PathPairing(path1, path2, recursive=recursive, maxdepth=maxdepth) + pairs = ( + await self._thread_pool.run(self._sync_fs._copy_pairs, pairing) + if pairing.expands + else pairing.copy_pairs() ) if not pairs: return @@ -856,7 +859,6 @@ async def _copy( [p2 for _, p2 in pairs], recursive=recursive, on_error=on_error, - maxdepth=maxdepth, batch_size=batch_size, **kwargs, ) @@ -920,7 +922,7 @@ async def _get( async def _get_pairs( self, pairs: list[tuple[str | S3Path, Any]], callback: Any, **kwargs - ) -> list[Any]: + ) -> None: """Download the pairs of a ``get()`` as fsspec's ``_get()`` downloads lists of paths. Each download gets a branch of the callback named after the string of @@ -932,9 +934,6 @@ async def _get_pairs( callback: Progress callback. **kwargs: Passed to ``_get_file()``, and ``batch_size``, the number of downloads to run at once. - - Returns: - The results of the downloads, as fsspec's ``_get()`` returns them. """ for _, lp in pairs: os.makedirs(os.path.dirname(lp), exist_ok=True) @@ -949,13 +948,10 @@ async def get_pair(rp: str | S3Path, lp: Any) -> None: lp: The local destination. """ with callback.branched(str(rp), lp, **kwargs) as child: - return await self._get_file(rp, lp, callback=child, **kwargs) + await self._get_file(rp, lp, callback=child, **kwargs) - return cast( - list[Any], - await _run_coros_in_chunks( - [get_pair(rp, lp) for rp, lp in pairs], batch_size=batch_size, callback=callback - ), + await _run_coros_in_chunks( + [get_pair(rp, lp) for rp, lp in pairs], batch_size=batch_size, callback=callback ) async def _put( @@ -1078,12 +1074,7 @@ async def _cat_paths( ) paths = list(targets) out = await _run_coros_in_chunks( - [ - self._cat_file(target, **kwargs) - if isinstance(target, str) - else self._cat_object(target, **kwargs) - for target in targets.values() - ], + [self._cat_file(target, **kwargs) for target in targets.values()], batch_size=batch_size or self.batch_size, nofiles=True, return_exceptions=True, @@ -1100,20 +1091,6 @@ async def _cat_paths( } return out[0] - async def _cat_object(self, path: S3Path, **kwargs) -> bytes: - """Read the contents of the object of a path, as ``_cat_file()`` does. - - Args: - path: The path of the object, whose bucket, key and version are - used as they are. - **kwargs: Additional parameters, as for ``_cat_file()``. - - Returns: - The bytes read from the object. - """ - _S3BatchTasks.track() - return await self._thread_pool.run(self._sync_fs._cat_object, path, **kwargs) - async def _du(self, path, total=True, maxdepth=None, **kwargs): """Return the sizes of the objects below a path, as fsspec's ``_du()`` does. @@ -1138,7 +1115,7 @@ async def _du(self, path, total=True, maxdepth=None, **kwargs): ) sizes: dict[str, int] = {} for _, target in found: - info = await self._thread_pool.run(self._sync_fs._target_info, target) + info = await self._info(target) sizes[info["name"]] = info["size"] return sum(sizes.values()) if total else sizes @@ -1211,8 +1188,11 @@ async def _cp_file( ValueError: If trying to copy to a versioned file or copy buckets. """ _S3BatchTasks.track() - path1, path2 = stringify_path(path1), stringify_path(path2) - await self._copy_file(path1, path2, **kwargs) + await self._copy_file( + path1 if isinstance(path1, S3Path) else stringify_path(path1), + path2 if isinstance(path2, S3Path) else stringify_path(path2), + **kwargs, + ) async def _copy_file(self, path1: str | S3Path, path2: str | S3Path, **kwargs) -> bool: """Copy an S3 object as :meth:`_cp_file` does. @@ -1232,8 +1212,8 @@ async def _copy_file(self, path1: str | S3Path, path2: str | S3Path, **kwargs) - # Parameters of the multipart copy, not of the S3 requests. block_size = kwargs.pop("block_size", None) max_workers = kwargs.pop("max_workers", None) - source = self._sync_fs._pair_path(path1) - destination = self._sync_fs._pair_path(path2) + source = S3Path.of(path1) + destination = S3Path.of(path2) if destination.version_id: raise ValueError("Cannot copy to a versioned file.") if not source.key or not destination.key: @@ -1241,16 +1221,7 @@ async def _copy_file(self, path1: str | S3Path, path2: str | S3Path, **kwargs) - # Looked up with the parameters that authorize the access to the # source; see S3FileSystem._copy_file. - source_params = self.core.copy_source_params(kwargs) - info1 = ( - await self._info(path1, **source_params) - if isinstance(path1, str) - else await self._thread_pool.run( - self._sync_fs._path_info, - source, - lookup_kwargs=self._sync_fs._get_lookup_kwargs(source_params), - ) - ) + info1 = await self._info(path1, **self.core.copy_source_params(kwargs)) if info1.get("type") == S3ObjectType.S3_OBJECT_TYPE_DIRECTORY: # Recursive copy() passes the directories too; see # S3FileSystem.cp_file. @@ -1269,10 +1240,7 @@ async def _copy_file(self, path1: str | S3Path, path2: str | S3Path, **kwargs) - ) finally: # See S3FileSystem._copy_file. - if isinstance(path2, str): - self._sync_fs.invalidate_cache(path2) - else: - self._sync_fs._invalidate(destination) + self._sync_fs._invalidate(destination) return True async def _copy_object_with_multipart_upload( diff --git a/pyathena/filesystem/s3_path.py b/pyathena/filesystem/s3_path.py index b8e490cc1..84249c9e4 100644 --- a/pyathena/filesystem/s3_path.py +++ b/pyathena/filesystem/s3_path.py @@ -88,6 +88,25 @@ def parse(cls, path: str) -> S3Path: raise ValueError(f"Invalid S3 path format {path}.") return cls(match.group("bucket"), match.group("key"), match.group("version_id")) + @classmethod + def of(cls, path: str | S3Path) -> S3Path: + """Return a path string parsed, or an S3 path as it is. + + An ``S3Path``, such as a key that a listing returns, is never parsed + again, so that a key that ends in what looks like a version ID query + keeps naming that key. + + Args: + path: The S3 path string, as for :meth:`parse`, or an ``S3Path``. + + Returns: + The path. + + Raises: + ValueError: If the string is not a valid S3 path. + """ + return path if isinstance(path, S3Path) else cls.parse(path) + @classmethod def split_version_id(cls, path: str) -> tuple[str, str | None]: """Split the version ID query from the end of a path string. diff --git a/pyathena/filesystem/s3_path_pairing.py b/pyathena/filesystem/s3_path_pairing.py index dd12a4fda..8083ff0bf 100644 --- a/pyathena/filesystem/s3_path_pairing.py +++ b/pyathena/filesystem/s3_path_pairing.py @@ -234,8 +234,8 @@ def conflict_candidates( TypeError: If ``versioning_enabled_buckets`` is a string. """ return [ - target if isinstance(p1, S3Path) else str(target) - for p1, target in self._moves(pairs, versioning_enabled_buckets, copy)[2] + pair.source if isinstance(pair.path1, S3Path) else str(pair.source) + for pair in self._moves(pairs, versioning_enabled_buckets, copy)[2] ] def move_pairs( @@ -341,8 +341,8 @@ def _check( if missing is None and candidates: raise ValueError("missing is needed to check the pairs.") # A directory without an object at its key writes no destination. - skipped = {target for _, target in candidates}.intersection( - self._target(self._as_path(path), versioning_enabled_buckets) for path in missing or () + skipped = {pair.source for pair in candidates}.intersection( + self._target(S3Path.of(path), versioning_enabled_buckets) for path in missing or () ) # A path with a version always names an object, so it writes its # destination even when the key has no current object. @@ -398,18 +398,6 @@ def _is_glob(path: str) -> bool: """ return has_magic(path) and not S3Path.has_version_id(path) - @staticmethod - def _as_path(path: str | S3Path) -> S3Path: - """Return a path of the pairs as an ``S3Path``. - - Args: - path: A path string, which is parsed, or an ``S3Path``. - - Returns: - The path. - """ - return path if isinstance(path, S3Path) else S3Path.parse(path) - @staticmethod def _target(path: S3Path, versioning_enabled_buckets: Collection[str]) -> S3Path: """Return a move target using the caller's bucket versioning state. @@ -429,7 +417,7 @@ def _moves( pairs: Sequence[tuple[str | S3Path, str | S3Path]], versioning_enabled_buckets: Collection[str] = (), copy: bool = False, - ) -> tuple[list[_NamedPair], set[S3Path], list[tuple[str | S3Path, S3Path]]]: + ) -> tuple[list[_NamedPair], set[S3Path], list[_NamedPair]]: """Compare the paths of a move by what they name. Args: @@ -440,9 +428,8 @@ def _moves( Returns: Each pair with whether its source has a version and the targets of its source and destination; the targets of all sources, - including those left in place; and the conflict candidates (see - :meth:`conflict_candidates`), each as the source of its pair and - its target. + including those left in place; and the pairs of the conflict + candidates (see :meth:`conflict_candidates`). Raises: TypeError: If ``versioning_enabled_buckets`` is a string. @@ -453,14 +440,14 @@ def _moves( ) named = [] for p1, p2 in pairs: - source_path = S3PathPairing._as_path(p1) + source_path = S3Path.of(p1) named.append( _NamedPair( p1, p2, bool(source_path.version_id), S3PathPairing._target(source_path, versioning_enabled_buckets), - S3PathPairing._target(S3PathPairing._as_path(p2), versioning_enabled_buckets), + S3PathPairing._target(S3Path.of(p2), versioning_enabled_buckets), ) ) # The sources left in place count too; a copy onto one overwrites it. @@ -475,7 +462,7 @@ def _moves( parent = parent.rpartition("/")[0] counts = Counter(pair.destination for pair in named if pair.source != pair.destination) candidates = [ - (pair.path1, pair.source) + pair for pair in named if pair.source != pair.destination and (counts[pair.destination] > 1 or pair.destination in sources) diff --git a/tests/pyathena/filesystem/test_s3.py b/tests/pyathena/filesystem/test_s3.py index f8ea014e2..0a729bb60 100644 --- a/tests/pyathena/filesystem/test_s3.py +++ b/tests/pyathena/filesystem/test_s3.py @@ -1319,6 +1319,56 @@ def test_get_downloads_listed_keys(self, tmp_path): ] assert sorted(callback.sources) == ["bucket/a", "bucket/a/x", "bucket/a/x?versionId=null"] + @pytest.mark.parametrize( + ("path", "version_id", "sent"), + [ + ( + S3Path("bucket", "a/x?versionId=null"), + None, + ("head_object", "a/x?versionId=null", None), + ), + ( + S3Path("bucket", "a/x?versionId=null"), + "v2", + ("head_object", "a/x?versionId=null", "v2"), + ), + ], + ) + def test_info_looks_up_path_as_it_is(self, path, version_id, sent): + # GH-1403: info() looks an S3Path up by its bucket, key and version, + # as given or from version_id, never by parsing its name. + fs = S3FileSystem( + key="dummy", secret="dummy", region_name="us-east-1", skip_instance_cache=True + ) + _, requests = serve_s3_keys(fs, self.NULL_KEYS) + assert fs.info(path, version_id=version_id)["name"] == "bucket/a/x?versionId=null" + assert s3_request_targets(requests) == [sent] + + @pytest.mark.parametrize( + ("call", "sent"), + [ + pytest.param( + lambda fs: fs.copy(["s3://bucket/a/x", "s3://bucket/a/*"], "s3://bucket/b/"), + [("copy_object", "b/x", None)], + id="copy", + ), + pytest.param( + lambda fs: fs.rm(["s3://bucket/a/x", "s3://bucket/a/*"]), + [("delete_objects", "a/x", None)], + id="rm", + ), + ], + ) + def test_expansion_acts_on_an_object_once(self, call, sent): + # GH-1403: a path that the caller passes and a listed key that name + # the same object are acted on once, as expand_path() names them once. + fs = S3FileSystem( + key="dummy", secret="dummy", region_name="us-east-1", skip_instance_cache=True + ) + _, requests = serve_s3_keys(fs, {"a/x"}) + call(fs) + assert [t for t in s3_request_targets(requests) if t[0] != "head_object"] == sent + def test_expand_path_expands_below_listed_matches(self): # GH-1402: a recursive glob expands below each match by its key prefix. fs = S3FileSystem( @@ -2775,7 +2825,7 @@ def test_mv_nothing_within_maxdepth(self): fs._expand_paths = mock.MagicMock( return_value=[("bucket/src/sub", S3Path("bucket", "src/sub"))] ) - fs._target_isdir = mock.MagicMock(return_value=True) + fs.isdir = mock.MagicMock(return_value=True) fs.mv("s3://bucket/src", "s3://bucket/dst/", recursive=True, maxdepth=1) fs._call.assert_not_called() diff --git a/tests/pyathena/filesystem/test_s3_path.py b/tests/pyathena/filesystem/test_s3_path.py index 8280d9b22..c72749c67 100644 --- a/tests/pyathena/filesystem/test_s3_path.py +++ b/tests/pyathena/filesystem/test_s3_path.py @@ -55,6 +55,13 @@ class TestS3Path: def test_parse(self, path, expected): assert S3Path.parse(path) == expected + def test_of(self): + # GH-1403: a string is parsed, and an S3Path, such as a listed key + # that ends in what looks like a version ID query, is kept as it is. + listed = S3Path("bucket", "a/x?versionId=null") + assert S3Path.of(listed) is listed + assert S3Path.of("s3://bucket/a/x?versionId=null") == S3Path("bucket", "a/x", "null") + def test_empty_key_is_the_bucket(self): # GH-1401: an empty key names the bucket, which the path prints as. assert S3Path("bucket", "") == S3Path("bucket")