diff --git a/docs/filesystem.md b/docs/filesystem.md index 85dcd236f..74c14d359 100644 --- a/docs/filesystem.md +++ b/docs/filesystem.md @@ -903,11 +903,17 @@ except Exception: ## Path pairing -`S3PathPairing` pairs the paths of one `copy()`, `get()` or `mv()`: `mv()` always, and -`copy()` and `get()` when a source has a version ID and the destination is one path -(fsspec pairs the others). Its `delete_paths()` splits the paths of an `rm()` into -those deleted as given and those expanded. The pairing is fsspec's, except that a -path with a version ID names that version, and its destination is named after its key. +`S3PathPairing` pairs the paths of one `copy()`, `get()` or `mv()`. Its `delete_paths()` +splits the paths of an `rm()` into those deleted as given and those expanded. The +pairing is fsspec's, except that a path with a version ID names that version, and its +destination is named after its key. + +A source may also be an `S3Path`, as the filesystems pass the keys that their listings +return. It names its bucket and key as they are, also a key that ends in what looks +like a version ID query, such as `a/x?versionId=null`, and its destination is named +after that key. `s3_copy_pairs()` pairs the sources as `copy_pairs()` does, and returns +the S3 destination of an `S3Path` source as an `S3Path`. `conflict_candidates()` returns +the candidates of `S3Path` sources as `S3Path` objects, and `missing` accepts them. A pairing is a frozen dataclass of `path1`, `path2`, `recursive` and `maxdepth`, and holds no filesystem. The filesystems read from it what to look up (`expands`, diff --git a/pyathena/filesystem/s3.py b/pyathena/filesystem/s3.py index 35fccee9b..0acd0515e 100644 --- a/pyathena/filesystem/s3.py +++ b/pyathena/filesystem/s3.py @@ -1162,7 +1162,7 @@ def _ls_directory( raise FileNotFoundError(directory.name) return list(files) - def info(self, path: str | os.PathLike[str], **kwargs) -> S3Object: + def info(self, path: str | os.PathLike[str] | S3Path, **kwargs) -> S3Object: """Return information about an S3 path. Uses the directory cache first: a cached entry for the path is @@ -1192,7 +1192,9 @@ def info(self, path: str | os.PathLike[str], **kwargs) -> S3Object: parameters are used. Args: - path: S3 path (e.g., "s3://bucket" or "s3://bucket/key"). + path: S3 path (e.g., "s3://bucket" or "s3://bucket/key"), or an + ``S3Path``, whose bucket, key and version are looked up as + they are, such as a key that a listing returns. **kwargs: Additional arguments including: refresh: If True, bypass the cache and query S3. version_id: The version ID to look up when the path has none. @@ -1208,24 +1210,27 @@ def info(self, path: str | os.PathLike[str], **kwargs) -> S3Object: FileNotFoundError: If the path does not exist. """ refresh = kwargs.pop("refresh", False) - path = self._strip_protocol(path) - if path in ["/", ""]: - # S3Path.parse rejects the root path. - return S3Object( - init={ - "ContentLength": 0, - "ContentType": None, - "StorageClass": S3StorageClass.S3_STORAGE_CLASS_BUCKET, - "ETag": None, - "LastModified": None, - }, - type=S3ObjectType.S3_OBJECT_TYPE_DIRECTORY, - bucket="", - key=None, - version_id=None, - ) - s3_path = S3Path.parse(path) version_id = kwargs.pop("version_id", None) + if isinstance(path, S3Path): + s3_path = path + else: + name = self._strip_protocol(path) + if name in ["/", ""]: + # S3Path.parse rejects the root path. + return S3Object( + init={ + "ContentLength": 0, + "ContentType": None, + "StorageClass": S3StorageClass.S3_STORAGE_CLASS_BUCKET, + "ETag": None, + "LastModified": None, + }, + type=S3ObjectType.S3_OBJECT_TYPE_DIRECTORY, + bucket="", + key=None, + version_id=None, + ) + s3_path = S3Path.parse(name) if version_id and not s3_path.version_id: s3_path = s3_path.with_version_id(version_id) return self._path_info( @@ -1775,25 +1780,10 @@ def du(self, path, total=True, maxdepth=None, withdirs=False, **kwargs): info = self.info(path) sizes[info["name"]] = info["size"] for _, target in self._find_paths(path, maxdepth=maxdepth, withdirs=withdirs, **kwargs): - info = self._target_info(target) + info = self.info(target) sizes[info["name"]] = info["size"] return sum(sizes.values()) if total else sizes - def _target_info(self, target: str | S3Path) -> S3Object: - """Look up a path that an expansion or ``find()`` names. - - Args: - target: A path string that the caller passed, which ``info()`` - looks up, or a listed path, looked up by its bucket and key. - - Returns: - S3Object describing the bucket, directory, or file. - - Raises: - FileNotFoundError: If the path does not exist. - """ - return self.info(target) if isinstance(target, str) else self._path_info(target) - def glob(self, path, maxdepth=None, **kwargs): """Find the paths that match a glob pattern. @@ -1966,7 +1956,9 @@ def _expand_paths( The names, as :meth:`expand_path` returns them, and the paths that they name, sorted by name: the name itself for a path that the caller passed, which the caller parses as a path string, and - an ``S3Path`` for a listed entry. + an ``S3Path`` for a listed entry. A path that the caller passed + and a listed entry that name the same object are returned once, + as the first of them. Raises: ValueError: If ``maxdepth`` is less than 1. @@ -1987,7 +1979,16 @@ def _expand_paths( raise if not out: raise FileNotFoundError(path) - return sorted(out, key=lambda item: item[0]) + # The first name and path of each object, by what the path names. + expanded: dict[S3Path | str, tuple[str, str | S3Path]] = {} + for name, target in out: + try: + identity: S3Path | str = S3Path.of(target) + except ValueError: + # Such as the root path, which names no bucket. + identity = target + expanded.setdefault(identity, (name, target)) + return sorted(expanded.values(), key=lambda item: item[0]) def _expand_unversioned( self, paths: list[str], recursive: bool, maxdepth: int | None, **kwargs @@ -2752,7 +2753,7 @@ def mv(self, path1, path2, recursive=False, maxdepth=None, batch_size=None, **kw concurrency = min(limit, len(pairs)) call_kwargs = self._batch_kwargs(concurrency, kwargs) - def copy_pair(pair: tuple[str, str]) -> bool: + def copy_pair(pair: tuple[str | S3Path, str | S3Path]) -> bool: """Copy a pair of the move. Args: @@ -2777,7 +2778,7 @@ def _move_pairs( recursive: bool = False, maxdepth: int | None = None, source_params: Mapping[str, Any] | None = None, - ) -> list[tuple[str, str]]: + ) -> list[tuple[str | S3Path, str | S3Path]]: """Pair and check the paths of a move (see :meth:`S3PathPairing.move_pairs`). Args: @@ -2807,10 +2808,10 @@ def _move_pairs( def _conflict_lookups( self, pairing: S3PathPairing, - pairs: list[tuple[str, str]], + pairs: list[tuple[str | S3Path, str | S3Path]], source_params: Mapping[str, Any] | None = None, copy: bool = False, - ) -> tuple[set[str], set[str]]: + ) -> tuple[set[str | S3Path], set[str]]: """Look up what the conflict checks of a move or a copy need. See :meth:`S3PathPairing.conflict_candidates`. @@ -2831,11 +2832,11 @@ def _conflict_lookups( Raises: OSError: If bucket versioning cannot be read. """ - paths = {p: S3Path.parse(p) for pair in pairs for p in pair} - unversioned = {p.name for p in paths.values() if not p.version_id} + paths = {S3Path.of(p) for pair in pairs for p in pair} + unversioned = {p.name for p in paths if not p.version_id} buckets = { p.bucket - for p in paths.values() + for p in paths if p.version_id == "null" and p.name in unversioned and not p.is_directory_bucket } versioned_buckets = { @@ -2847,44 +2848,51 @@ def _conflict_lookups( for source in pairing.conflict_candidates( pairs, versioning_enabled_buckets=versioned_buckets, copy=copy ) - if self._head_object(S3Path.parse(source), lookup_kwargs=lookup_kwargs) is None + if self._head_object(S3Path.of(source), lookup_kwargs=lookup_kwargs) is None } return missing, versioned_buckets def _copy_pairs( self, pairing: S3PathPairing, - isdir: Callable[[str], bool] | None = None, + local: bool = False, **kwargs: Any, - ) -> list[tuple[str, str]]: - """Expand and pair the paths of a copy (see :meth:`S3PathPairing.copy_pairs`). + ) -> list[tuple[str | S3Path, str | S3Path]]: + """Expand and pair the paths of a copy (see :meth:`S3PathPairing.s3_copy_pairs`). - The destination is looked up only when it decides the pairing. + The sources are expanded as :meth:`_expand_paths` names them: a path + that the caller passes as the string it was given, and a listed entry + as its bucket and key. The destination is looked up only when it + decides the pairing. Args: pairing: The paths of the copy. - isdir: Whether the destination is a directory, by default - ``self.isdir``; ``get()`` passes the local filesystem's. - **kwargs: Passed to ``expand_path()``, as fsspec's ``get()`` passes - its keyword arguments. + local: Whether the destinations are the local paths of ``get()``, + which the local filesystem looks up, and which are named as + :meth:`S3PathPairing.copy_pairs` names them. + **kwargs: Passed to the expansion, as fsspec's ``get()`` passes its + keyword arguments to ``expand_path()``. Returns: The sources and their destinations. """ if not pairing.expands: - return pairing.copy_pairs() - sources = self.expand_path( - pairing.path1, recursive=pairing.recursive, maxdepth=pairing.maxdepth, **kwargs - ) - if pairing.skips_directories: - sources = [p for p in sources if not (trailing_sep(p) or self.isdir(p))] + return list(pairing.copy_pairs()) + sources = [ + target + for name, target in self._expand_paths( + pairing.path1, recursive=pairing.recursive, maxdepth=pairing.maxdepth, **kwargs + ) + if not (pairing.skips_directories and (trailing_sep(name) or self.isdir(target))) + ] + isdir = LocalFileSystem().isdir if local else self.isdir destination_is_dir = ( # A string, as looks_up_destination checks. - (isdir or self.isdir)(cast(str, pairing.path2)) - if sources and pairing.looks_up_destination - else None + isdir(cast(str, pairing.path2)) if sources and pairing.looks_up_destination else None ) - return pairing.copy_pairs(sources, destination_is_dir) + if local: + return list(pairing.copy_pairs(sources, destination_is_dir)) + return pairing.s3_copy_pairs(sources, destination_is_dir) def copy( self, @@ -2951,7 +2959,7 @@ def copy( concurrency = min(limit, len(pairs)) call_kwargs = self._batch_kwargs(concurrency, kwargs) - def copy_pair(pair: tuple[str, str]) -> None: + def copy_pair(pair: tuple[str | S3Path, str | S3Path]) -> None: """Copy a pair. Args: @@ -3011,7 +3019,7 @@ def get( root = make_path_posix(lpath) if isinstance(lpath, (str, os.PathLike)) else lpath pairs = self._copy_pairs( S3PathPairing(rpath, root, recursive=recursive, maxdepth=maxdepth), - isdir=LocalFileSystem().isdir, + local=True, **kwargs, ) if isinstance(root, str): @@ -3026,14 +3034,15 @@ def get( call_kwargs = self._batch_kwargs(concurrency, kwargs) locked = _S3LockedCallback(callback, threading.Lock()) - def get_pair(pair: tuple[str, Any]) -> None: + def get_pair(pair: tuple[str | S3Path, Any]) -> None: """Download a pair with a branch of the callback. Args: - pair: The S3 source and its local destination. + pair: The S3 source and its local destination. The branch + is named after the source as a string. """ rp, lp = pair - with locked.branched(rp, lp) as child: + with locked.branched(str(rp), lp) as child: self.get_file(rp, lp, callback=child, **call_kwargs) locked.relative_update(1) @@ -3082,8 +3091,8 @@ def _check_local_destinations(destinations: list[Any]) -> None: def cp_file( self, - path1: str | os.PathLike[str], - path2: str | os.PathLike[str], + path1: str | os.PathLike[str] | S3Path, + path2: str | os.PathLike[str] | S3Path, recursive=False, maxdepth=None, on_error=None, @@ -3096,8 +3105,10 @@ def cp_file( and multipart copy based on object size. Args: - path1: Source S3 path (s3://bucket/key). - path2: Destination S3 path (s3://bucket/key). + path1: Source S3 path (s3://bucket/key), or an ``S3Path``, whose + bucket, key and version are used as they are, such as a key + that a listing returns. + path2: Destination S3 path (s3://bucket/key), or an ``S3Path``. recursive: Unused parameter for fsspec compatibility. maxdepth: Unused parameter for fsspec compatibility. on_error: Unused parameter for fsspec compatibility. @@ -3122,15 +3133,18 @@ def cp_file( A directory ``path1``, which recursive ``copy()`` passes along with the files under it, is skipped. """ - path1, path2 = stringify_path(path1), stringify_path(path2) - self._copy_file(path1, path2, **kwargs) + self._copy_file( + path1 if isinstance(path1, S3Path) else stringify_path(path1), + path2 if isinstance(path2, S3Path) else stringify_path(path2), + **kwargs, + ) - def _copy_file(self, path1: str, path2: str, **kwargs) -> bool: + def _copy_file(self, path1: str | S3Path, path2: str | S3Path, **kwargs) -> bool: """Copy an S3 object as :meth:`cp_file` does. Args: - path1: Source S3 path (s3://bucket/key). - path2: Destination S3 path (s3://bucket/key). + path1: Source S3 path (s3://bucket/key), or an ``S3Path``. + path2: Destination S3 path (s3://bucket/key), or an ``S3Path``. **kwargs: Additional S3 copy parameters, as for :meth:`cp_file`. Returns: @@ -3143,8 +3157,8 @@ def _copy_file(self, path1: str, path2: str, **kwargs) -> bool: # Parameters of the multipart copy, not of the S3 requests. block_size = kwargs.pop("block_size", None) max_workers = kwargs.pop("max_workers", None) - source = S3Path.parse(path1) - destination = S3Path.parse(path2) + source = S3Path.of(path1) + destination = S3Path.of(path2) if destination.version_id: raise ValueError("Cannot copy to a versioned file.") if not source.key or not destination.key: @@ -3173,7 +3187,7 @@ def _copy_file(self, path1: str, path2: str, **kwargs) -> bool: finally: # A multipart copy that fails to copy an annotation has already # written the destination. - self.invalidate_cache(path2) + self._invalidate(destination) return True def _copy_object_with_multipart_upload( @@ -3442,13 +3456,13 @@ def cat(self, path, recursive=False, on_error="raise", batch_size=None, **kwargs targets = self._first_targets(self._expand_paths(path, recursive=recursive, **kwargs)) paths = list(targets) if not (len(paths) > 1 or isinstance(path, list) or paths[0] != self._strip_protocol(path)): - return self._cat_target(targets[paths[0]], **kwargs) + return self.cat_file(targets[paths[0]], **kwargs) outcomes: list[Any] | None = None contents: dict[str, Any] | None = None outcome = None try: outcomes = self._run_batch( - functools.partial(self._cat_target, **kwargs), + functools.partial(self.cat_file, **kwargs), list(targets.values()), min(limit, len(paths)), counts=None if on_error == "raise" else lambda e: False, @@ -3485,22 +3499,6 @@ def _first_targets(expanded: list[tuple[str, str | S3Path]]) -> dict[str, str | targets.setdefault(name, target) return targets - def _cat_target(self, target: str | S3Path, **kwargs) -> bytes: - """Read a path of an expansion. - - Args: - target: A path string that the caller passed, which - :meth:`cat_file` reads, or a listed path, read by its bucket - and key. - **kwargs: Additional parameters, as for :meth:`cat_file`. - - Returns: - The bytes read from the object. - """ - if isinstance(target, str): - return self.cat_file(target, **kwargs) - return self._cat_object(target, **kwargs) - def cat_ranges( self, paths, starts, ends, max_gap=None, on_error="return", batch_size=None, **kwargs ) -> list[bytes | Exception]: @@ -3568,7 +3566,7 @@ def cat_range(item: tuple[str, int | None, int | None]) -> bytes: def cat_file( self, - path: str | os.PathLike[str], + path: str | os.PathLike[str] | S3Path, start: int | None = None, end: int | None = None, **kwargs, @@ -3585,7 +3583,9 @@ def cat_file( range and receives the lookup parameters among ``kwargs``. Args: - path: S3 path (s3://bucket/key) of the object. + path: S3 path (s3://bucket/key) of the object, or an ``S3Path``, + whose bucket, key and version are read as they are, such as a + key that a listing returns. start: Byte offset to start reading at. A negative value counts from the end of the object. end: Byte offset to stop reading at (exclusive). A negative value @@ -3601,6 +3601,8 @@ def cat_file( FileNotFoundError: If the path has no key or the key does not exist. """ + if isinstance(path, S3Path): + return self._cat_object(path, start=start, end=end, **kwargs) path = stringify_path(path) return self._cat_object(S3Path.parse(path), start=start, end=end, name=path, **kwargs) @@ -3651,12 +3653,8 @@ def _cat_object( # empty range sends no GetObject request that would report a # missing object. lookup_kwargs = self._get_lookup_kwargs(kwargs) - info = ( - self.info(name, version_id=version_id, **lookup_kwargs) - if name is not None - else self._path_info( - path.with_version_id(version_id), lookup_kwargs=lookup_kwargs - ) + info = self.info( + name if name is not None else path, version_id=version_id, **lookup_kwargs ) if ( info.get("type") == S3ObjectType.S3_OBJECT_TYPE_DIRECTORY @@ -3855,7 +3853,7 @@ def put_file( def get_file( self, - rpath: str | os.PathLike[str], + rpath: str | os.PathLike[str] | S3Path, lpath=None, callback=_DEFAULT_CALLBACK, outfile=None, @@ -3883,7 +3881,9 @@ def get_file( directories of a local file ``lpath`` are created as needed. Args: - rpath: S3 source path (s3://bucket/key). + rpath: S3 source path (s3://bucket/key), or an ``S3Path``, whose + bucket, key and version are used as they are, such as a key + that a listing returns. lpath: Local destination path, or a file-like object to write to. Not needed when ``outfile`` is given. callback: Progress callback for tracking download progress, with @@ -3899,12 +3899,12 @@ def get_file( ``DEFAULT_BLOCK_SIZE`` if that is not positive either, and the size of the callback is unknown. """ - path_version_id = S3Path.parse(self._strip_protocol(rpath)).version_id + s3_path = S3Path.of(rpath if isinstance(rpath, S3Path) else self._strip_protocol(rpath)) if outfile is None and isfilelike(lpath): outfile = lpath elif ( outfile is None - and not (path_version_id or kwargs.get("version_id")) + and not (s3_path.version_id or kwargs.get("version_id")) and self.isdir(rpath) ): # A requested version always names an object, while isdir() @@ -3915,9 +3915,7 @@ def get_file( # The codec, if any, as fsspec's open() resolves it, with "infer" # from the key without the version. - codec = get_compression( - S3Path.parse(self._strip_protocol(rpath)).key or "", kwargs.pop("compression", None) - ) + codec = get_compression(s3_path.key or "", kwargs.pop("compression", None)) # Not the caller's cache: one such as "all" would read the whole # object when the file is opened, and "background" reads outside # the stop scopes of the caller. Without a codec, the object is read @@ -3934,7 +3932,11 @@ def get_file( # The remote file is opened first so that no local file is created # when open() finds no object at the path. with contextlib.ExitStack() as stack: - remote = stack.enter_context(self.open(rpath, "rb", **kwargs)) + remote = stack.enter_context( + self._open(rpath, "rb", **kwargs) + if isinstance(rpath, S3Path) + else self.open(rpath, "rb", **kwargs) + ) reader: Any = None if codec is not None: # As fsspec's open() wraps the file, but closed with it. @@ -4488,14 +4490,14 @@ def _key_exists(self, path: S3Path, lookup_kwargs: Mapping[str, Any]) -> bool: True if an object exists at the key. """ if not (path.key or "").endswith("/"): - return self._object_exists(path.uri, lookup_kwargs) + return self._object_exists(path, lookup_kwargs) try: self.core.head_object(path, **lookup_kwargs) except FileNotFoundError: return False return True - def _object_exists(self, path: str, lookup_kwargs: Mapping[str, Any]) -> bool: + def _object_exists(self, path: str | S3Path, lookup_kwargs: Mapping[str, Any]) -> bool: """Check whether an object exists at the path, for a write that must not replace one. A key that is only a prefix of other keys has no object, as when @@ -4660,7 +4662,7 @@ def _ls_from_cache(self, path: S3Path) -> list[S3Object] | S3Object | None: def _open( self, - path: str, + path: str | S3Path, mode: str = "rb", block_size: int | None = None, cache_type: str | None = None, @@ -4777,7 +4779,7 @@ class S3FileTarget: def resolve( cls, fs: S3FileSystem, - path: str, + path: str | S3Path, mode: str, *, block_size: int, @@ -4791,7 +4793,10 @@ def resolve( Args: fs: The filesystem that the file belongs to. - path: S3 path (s3://bucket/key) of the file. + path: S3 path (s3://bucket/key) of the file, or an ``S3Path``, + whose bucket, key and version are used as they are, such as + a key that a listing returns, and which is looked up by them. + The path of the file is then the string of the ``S3Path``. mode: The file mode: ``rb``, ``wb``, ``ab``, or ``xb``. block_size: The block size of the file. version_id: The version ID to read. Must match the version ID in @@ -4816,7 +4821,11 @@ def resolve( """ # A new dictionary, so that the caller's is not modified. kwargs: dict[str, Any] = dict(request_kwargs or {}) - s3_path = S3Path.parse(path) + # The path as it is looked up: a path string as a string, and an + # S3Path by its bucket, key and version. + target = path + s3_path = S3Path.of(path) + path = str(path) path_version_id = s3_path.version_id if not s3_path.key: raise ValueError("The path does not contain a key.") @@ -4834,6 +4843,7 @@ def resolve( # Carry the version in the path, as with the ?versionId= suffix, # so that a reopened (e.g., unpickled) file reads the same version. path = f"{path}?versionId={version_id}" + target = s3_path.with_version_id(version_id) if isinstance(target, S3Path) else path if "r" not in mode and not ( fs.core.MULTIPART_UPLOAD_MIN_PART_SIZE <= block_size @@ -4856,7 +4866,7 @@ def resolve( if "r" in mode: # Looked up here; the base class initializer of a file would # otherwise take the size from the latest version of the object. - info = fs.info(path, version_id=version_id, **lookup_kwargs) + info = fs.info(target, version_id=version_id, **lookup_kwargs) if info.get("type") == S3ObjectType.S3_OBJECT_TYPE_DIRECTORY: # A prefix has no object to read. raise FileNotFoundError(path) @@ -4880,7 +4890,7 @@ def resolve( # The rewritten object keeps the metadata of the existing one, # which a cached listing entry lacks, so look up the object. with contextlib.suppress(FileNotFoundError): - appended = fs.info(path, refresh=True, **lookup_kwargs) + appended = fs.info(target, refresh=True, **lookup_kwargs) if ( appended is not None and appended.get("type") == S3ObjectType.S3_OBJECT_TYPE_DIRECTORY @@ -4896,7 +4906,7 @@ def resolve( # the whole object is read, and an object replaced since # is not. condition = {"IfMatch": etag} if etag else {} - append_data = fs.cat_file(path, **condition, **lookup_kwargs) + append_data = fs.cat_file(target, **condition, **lookup_kwargs) # The attributes of the object looked up are kept where the # caller gives none: the encryption as a whole, so that its # fields are not combined with another encryption, and @@ -4919,7 +4929,7 @@ def resolve( elif "x" in mode: # Checked up front so that no data is uploaded for an existing # object, and on commit with IfNoneMatch for one created since. - if fs._object_exists(path, lookup_kwargs): + if fs._object_exists(target, lookup_kwargs): raise FileExistsError(path) kwargs.update({"IfNoneMatch": "*"}) return cls( @@ -4947,7 +4957,7 @@ class S3File(AbstractBufferedFile): def __init__( self, fs: S3FileSystem, - path: str, + path: str | S3Path, mode: str = "rb", version_id: str | None = None, max_workers: int | None = None, @@ -4974,7 +4984,8 @@ def __init__( Args: fs: The filesystem that the file belongs to. - path: S3 path (s3://bucket/key) of the file. + path: S3 path (s3://bucket/key) of the file, or an ``S3Path`` + (see :meth:`S3FileTarget.resolve`). mode: The file mode: ``rb``, ``wb``, ``ab``, or ``xb``. version_id: The version ID to read. Must match the version ID in the path if both are given. A version cannot be given, in diff --git a/pyathena/filesystem/s3_async.py b/pyathena/filesystem/s3_async.py index 9dde3240c..2cc9a7931 100644 --- a/pyathena/filesystem/s3_async.py +++ b/pyathena/filesystem/s3_async.py @@ -32,7 +32,7 @@ ) from fsspec.callbacks import _DEFAULT_CALLBACK from fsspec.core import get_compression -from fsspec.implementations.local import LocalFileSystem, make_path_posix +from fsspec.implementations.local import make_path_posix from fsspec.utils import check_contained, stringify_path from pyathena.aio.util import create_in_thread @@ -435,7 +435,7 @@ async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: """Release the threads with :meth:`close`.""" self.close() - async def _info(self, path: str | os.PathLike[str], **kwargs) -> S3Object: + async def _info(self, path: str | os.PathLike[str] | S3Path, **kwargs) -> S3Object: return await self._thread_pool.run(self._sync_fs.info, path, **kwargs) async def _ls( @@ -445,7 +445,7 @@ async def _ls( async def _cat_file( self, - path: str | os.PathLike[str], + path: str | os.PathLike[str] | S3Path, start: int | None = None, end: int | None = None, **kwargs, @@ -623,7 +623,7 @@ def _put_file_in_transaction( async def _get_file( self, - rpath: str | os.PathLike[str], + rpath: str | os.PathLike[str] | S3Path, lpath: str | os.PathLike[str], callback=_DEFAULT_CALLBACK, **kwargs, @@ -824,11 +824,12 @@ async def _copy( ) -> None: """Copy files within S3. - See :meth:`S3FileSystem.copy`. The copies run as in fsspec's - ``_copy()``, ``batch_size`` of them at once as tasks. A cancellation - cancels the copies and waits for them; a copy that is already being - cancelled, such as by the shutdown of the event loop, is not - cancelled again. + See :meth:`S3FileSystem.copy`. The paths are paired as there, but the + pairs are not checked for conflicts, and the copies run as in + fsspec's ``_copy()``, ``batch_size`` of them at once as tasks. A + cancellation cancels the copies and waits for them; a copy that is + already being cancelled, such as by the shutdown of the event loop, + is not cancelled again. Args: path1: Source S3 path, glob pattern, or list of them. @@ -841,22 +842,23 @@ async def _copy( **kwargs: Additional S3 copy parameters passed to ``_cp_file()``. """ path1, path2 = self._sync_fs._stringify_paths(path1), self._sync_fs._stringify_paths(path2) - sources = [path1] if isinstance(path1, str) else path1 - if isinstance(path2, str) and any(S3Path.has_version_id(p) for p in sources): - pairs = await self._thread_pool.run( - self._sync_fs._copy_pairs, - S3PathPairing(path1, path2, recursive=recursive, maxdepth=maxdepth), - ) - if not pairs: - return - path1, path2 = [p1 for p1, _ in pairs], [p2 for _, p2 in pairs] + # Paired as the sync copy() pairs them, so that the keys that a + # listing returns are copied by their buckets and keys; fsspec copies + # the lists of the pairs. + pairing = S3PathPairing(path1, path2, recursive=recursive, maxdepth=maxdepth) + pairs = ( + await self._thread_pool.run(self._sync_fs._copy_pairs, pairing) + if pairing.expands + else pairing.copy_pairs() + ) + if not pairs: + return await _S3BatchTasks().run( super()._copy( - path1, - path2, + [p1 for p1, _ in pairs], + [p2 for _, p2 in pairs], recursive=recursive, on_error=on_error, - maxdepth=maxdepth, batch_size=batch_size, **kwargs, ) @@ -901,7 +903,7 @@ async def _get( pairs = await self._thread_pool.run( self._sync_fs._copy_pairs, S3PathPairing(rpath, root, recursive=recursive, maxdepth=maxdepth), - isdir=LocalFileSystem().isdir, + local=True, ) if isinstance(root, str): check_contained(root, [lp for _, lp in pairs]) @@ -915,12 +917,41 @@ async def _get( if batch_size is not None and batch_size > 0: concurrency = min(concurrency, batch_size) await _S3BatchTasks().run( - super()._get( - [rp for rp, _ in pairs], - [lp for _, lp in pairs], - callback=callback, - **self._sync_fs._batch_kwargs(concurrency, kwargs), - ) + self._get_pairs(pairs, callback, **self._sync_fs._batch_kwargs(concurrency, kwargs)) + ) + + async def _get_pairs( + self, pairs: list[tuple[str | S3Path, Any]], callback: Any, **kwargs + ) -> None: + """Download the pairs of a ``get()`` as fsspec's ``_get()`` downloads lists of paths. + + Each download gets a branch of the callback named after the string of + its source, and the sources that a listing returns are downloaded by + their buckets and keys. + + Args: + pairs: The S3 sources and their local destinations. + callback: Progress callback. + **kwargs: Passed to ``_get_file()``, and ``batch_size``, the + number of downloads to run at once. + """ + for _, lp in pairs: + os.makedirs(os.path.dirname(lp), exist_ok=True) + batch_size = kwargs.pop("batch_size", self.batch_size) + callback.set_size(len(pairs)) + + async def get_pair(rp: str | S3Path, lp: Any) -> None: + """Download a pair with a branch of the callback. + + Args: + rp: The S3 source. + lp: The local destination. + """ + with callback.branched(str(rp), lp, **kwargs) as child: + await self._get_file(rp, lp, callback=child, **kwargs) + + await _run_coros_in_chunks( + [get_pair(rp, lp) for rp, lp in pairs], batch_size=batch_size, callback=callback ) async def _put( @@ -1043,12 +1074,7 @@ async def _cat_paths( ) paths = list(targets) out = await _run_coros_in_chunks( - [ - self._cat_file(target, **kwargs) - if isinstance(target, str) - else self._cat_object(target, **kwargs) - for target in targets.values() - ], + [self._cat_file(target, **kwargs) for target in targets.values()], batch_size=batch_size or self.batch_size, nofiles=True, return_exceptions=True, @@ -1065,20 +1091,6 @@ async def _cat_paths( } return out[0] - async def _cat_object(self, path: S3Path, **kwargs) -> bytes: - """Read the contents of the object of a path, as ``_cat_file()`` does. - - Args: - path: The path of the object, whose bucket, key and version are - used as they are. - **kwargs: Additional parameters, as for ``_cat_file()``. - - Returns: - The bytes read from the object. - """ - _S3BatchTasks.track() - return await self._thread_pool.run(self._sync_fs._cat_object, path, **kwargs) - async def _du(self, path, total=True, maxdepth=None, **kwargs): """Return the sizes of the objects below a path, as fsspec's ``_du()`` does. @@ -1103,7 +1115,7 @@ async def _du(self, path, total=True, maxdepth=None, **kwargs): ) sizes: dict[str, int] = {} for _, target in found: - info = await self._thread_pool.run(self._sync_fs._target_info, target) + info = await self._info(target) sizes[info["name"]] = info["size"] return sum(sizes.values()) if total else sizes @@ -1150,8 +1162,8 @@ async def _cat_ranges( async def _cp_file( self, - path1: str | os.PathLike[str], - path2: str | os.PathLike[str], + path1: str | os.PathLike[str] | S3Path, + path2: str | os.PathLike[str] | S3Path, recursive=False, maxdepth=None, on_error=None, @@ -1163,8 +1175,9 @@ async def _cp_file( the files under it, is skipped. Args: - path1: Source S3 path (s3://bucket/key). - path2: Destination S3 path (s3://bucket/key). + path1: Source S3 path (s3://bucket/key), or an ``S3Path``, as + for :meth:`S3FileSystem.cp_file`. + path2: Destination S3 path (s3://bucket/key), or an ``S3Path``. recursive: Unused parameter for fsspec compatibility. maxdepth: Unused parameter for fsspec compatibility. on_error: Unused parameter for fsspec compatibility. @@ -1175,15 +1188,18 @@ async def _cp_file( ValueError: If trying to copy to a versioned file or copy buckets. """ _S3BatchTasks.track() - path1, path2 = stringify_path(path1), stringify_path(path2) - await self._copy_file(path1, path2, **kwargs) + await self._copy_file( + path1 if isinstance(path1, S3Path) else stringify_path(path1), + path2 if isinstance(path2, S3Path) else stringify_path(path2), + **kwargs, + ) - async def _copy_file(self, path1: str, path2: str, **kwargs) -> bool: + async def _copy_file(self, path1: str | S3Path, path2: str | S3Path, **kwargs) -> bool: """Copy an S3 object as :meth:`_cp_file` does. Args: - path1: Source S3 path (s3://bucket/key). - path2: Destination S3 path (s3://bucket/key). + path1: Source S3 path (s3://bucket/key), or an ``S3Path``. + path2: Destination S3 path (s3://bucket/key), or an ``S3Path``. **kwargs: Additional S3 copy parameters, as for :meth:`_cp_file`. Returns: @@ -1196,8 +1212,8 @@ async def _copy_file(self, path1: str, path2: str, **kwargs) -> bool: # Parameters of the multipart copy, not of the S3 requests. block_size = kwargs.pop("block_size", None) max_workers = kwargs.pop("max_workers", None) - source = S3Path.parse(path1) - destination = S3Path.parse(path2) + source = S3Path.of(path1) + destination = S3Path.of(path2) if destination.version_id: raise ValueError("Cannot copy to a versioned file.") if not source.key or not destination.key: @@ -1224,7 +1240,7 @@ async def _copy_file(self, path1: str, path2: str, **kwargs) -> bool: ) finally: # See S3FileSystem._copy_file. - self._sync_fs.invalidate_cache(path2) + self._sync_fs._invalidate(destination) return True async def _copy_object_with_multipart_upload( diff --git a/pyathena/filesystem/s3_path.py b/pyathena/filesystem/s3_path.py index b8e490cc1..84249c9e4 100644 --- a/pyathena/filesystem/s3_path.py +++ b/pyathena/filesystem/s3_path.py @@ -88,6 +88,25 @@ def parse(cls, path: str) -> S3Path: raise ValueError(f"Invalid S3 path format {path}.") return cls(match.group("bucket"), match.group("key"), match.group("version_id")) + @classmethod + def of(cls, path: str | S3Path) -> S3Path: + """Return a path string parsed, or an S3 path as it is. + + An ``S3Path``, such as a key that a listing returns, is never parsed + again, so that a key that ends in what looks like a version ID query + keeps naming that key. + + Args: + path: The S3 path string, as for :meth:`parse`, or an ``S3Path``. + + Returns: + The path. + + Raises: + ValueError: If the string is not a valid S3 path. + """ + return path if isinstance(path, S3Path) else cls.parse(path) + @classmethod def split_version_id(cls, path: str) -> tuple[str, str | None]: """Split the version ID query from the end of a path string. diff --git a/pyathena/filesystem/s3_path_pairing.py b/pyathena/filesystem/s3_path_pairing.py index 68e4a6a0a..8083ff0bf 100644 --- a/pyathena/filesystem/s3_path_pairing.py +++ b/pyathena/filesystem/s3_path_pairing.py @@ -27,10 +27,14 @@ class S3PathPairing: The sources are paired with their destinations as fsspec's ``copy()`` pairs them, except that a path with a version ID names that version of an object: it is not a glob pattern, and its destination is named after its - key without the version. A move compares the paths by what they name - (see :attr:`~pyathena.filesystem.s3_path.S3Path.target`), keeping ``null`` - versions distinct from their keys in the caller's versioning-enabled - buckets. + key without the version. A source may also be an + :class:`~pyathena.filesystem.s3_path.S3Path`, such as a key that a listing + returns, which names its bucket and key as they are, also when the key + ends in what looks like a version ID query, and which is never parsed + again (see :meth:`s3_copy_pairs`). A move compares the paths by what they + name (see :attr:`~pyathena.filesystem.s3_path.S3Path.target`), keeping + ``null`` versions distinct from their keys in the caller's + versioning-enabled buckets. :meth:`delete_paths` splits the paths of an ``rm()``. The pairing is a frozen dataclass of the paths as given, and holds no @@ -97,11 +101,15 @@ def looks_up_destination(self) -> bool: def copy_pairs( self, - sources: Sequence[str] | None = None, + sources: Sequence[str | S3Path] | None = None, destination_is_dir: bool | None = None, - ) -> list[tuple[str, str]]: + ) -> list[tuple[str | S3Path, str]]: """Pair the sources with their destinations as fsspec's ``copy()`` does. + A source string with a version ID is named after its key without the + version, and an ``S3Path`` source after its bucket and key (its + ``name``), whose suffix the destination keeps as it is. + Args: sources: The expansion of ``path1``, without the directories that :attr:`skips_directories` leaves out; needed when @@ -136,17 +144,64 @@ def copy_pairs( exists = source_is_str and ( (glob and len(sources) == 1) or (not glob and dest_is_dir and not trailing_sep(path1)) ) - names = [S3Path.split_version_id(p)[0] for p in sources] + names = [ + p.name if isinstance(p, S3Path) else S3Path.split_version_id(p)[0] for p in sources + ] destinations = other_paths(names, path2, exists=exists, flatten=not source_is_str) return list(zip(sources, destinations, strict=True)) + def s3_copy_pairs( + self, + sources: Sequence[str | S3Path] | None = None, + destination_is_dir: bool | None = None, + ) -> list[tuple[str | S3Path, str | S3Path]]: + """Pair the sources with their S3 destinations, as :meth:`copy_pairs` does. + + The destination of an ``S3Path`` source, when ``path2`` is a string, + is an ``S3Path`` too: ``path2`` parsed, with the suffix that + :meth:`copy_pairs` appends to it added to its key as it is, so that a + key such as ``a/x?versionId=null`` is copied to the key + ``b/x?versionId=null`` instead of being parsed as a version. The + other destinations are strings, as :meth:`copy_pairs` returns them. + + Args: + sources: As for :meth:`copy_pairs`. + destination_is_dir: As for :meth:`copy_pairs`. + + Returns: + The sources and their destinations. + + Raises: + ValueError: If ``sources`` or ``destination_is_dir`` is needed and + None, or if ``path2`` names a version, which cannot be + written, and a destination is an ``S3Path``. + """ + pairs = self.copy_pairs(sources, destination_is_dir) + if not isinstance(self.path2, str) or not any(isinstance(p1, S3Path) for p1, _ in pairs): + return list(pairs) + # Every destination that copy_pairs() names after a source starts with + # path2 without its trailing slashes, followed by the suffix. + root_name = self.path2.rstrip("/") + root = S3Path.parse(root_name) + if root.version_id: + raise ValueError("Cannot copy to a versioned file.") + typed: list[tuple[str | S3Path, str | S3Path]] = [] + for p1, p2 in pairs: + if isinstance(p1, S3Path): + suffix = p2[len(root_name) :] + key = f"{root.key}{suffix}" if root.key else suffix.removeprefix("/") + typed.append((p1, S3Path(root.bucket, key))) + else: + typed.append((p1, p2)) + return typed + def conflict_candidates( self, - pairs: Sequence[tuple[str, str]], + pairs: Sequence[tuple[str | S3Path, str | S3Path]], *, versioning_enabled_buckets: Collection[str] = (), copy: bool = False, - ) -> list[str]: + ) -> list[str | S3Path]: """Return the sources of a move whose conflicts depend on an object at their key. A source with another source below it may be a directory. If no object @@ -158,7 +213,9 @@ def conflict_candidates( Args: pairs: The sources and destinations of the move, as - :meth:`copy_pairs` of this pairing returns them. + :meth:`copy_pairs` or :meth:`s3_copy_pairs` of this pairing + returns them. A string is parsed; an ``S3Path`` is used as + it is. versioning_enabled_buckets: Buckets whose versioning is enabled. Their ``null`` versions are distinct from their unversioned keys. Use the same collection for :meth:`move_pairs`. @@ -167,32 +224,36 @@ def conflict_candidates( conflicts. Returns: - The sources, in ``bucket/key`` form (their - :attr:`~pyathena.filesystem.s3_path.S3Path.target`), that have - another source below them and a destination that conflicts, one - per pair in the order of the pairs; empty if nothing needs to be - looked up. + The sources that have another source below them and a + destination that conflicts, one per pair in the order of the + pairs, as their :attr:`~pyathena.filesystem.s3_path.S3Path.target`: + in ``bucket/key`` form for a source string, and as an ``S3Path`` + for an ``S3Path`` source. Empty if nothing needs to be looked up. Raises: TypeError: If ``versioning_enabled_buckets`` is a string. """ - return self._moves(pairs, versioning_enabled_buckets, copy)[2] + return [ + pair.source if isinstance(pair.path1, S3Path) else str(pair.source) + for pair in self._moves(pairs, versioning_enabled_buckets, copy)[2] + ] def move_pairs( self, - pairs: Sequence[tuple[str, str]], - missing: Collection[str] | None = None, + pairs: Sequence[tuple[str | S3Path, str | S3Path]], + missing: Collection[str | S3Path] | None = None, *, versioning_enabled_buckets: Collection[str] = (), - ) -> list[tuple[str, str]]: + ) -> list[tuple[str | S3Path, str | S3Path]]: """Check the pairs of a move and leave out the sources that stay in place. Args: pairs: The sources and destinations of the move, as - :meth:`copy_pairs` of this pairing returns them. + :meth:`copy_pairs` or :meth:`s3_copy_pairs` of this pairing + returns them. missing: The :meth:`conflict_candidates` without an object at their - key, in any form that names them; needed when there are - candidates. + key, as strings in any form that names them or as ``S3Path`` + objects; needed when there are candidates. versioning_enabled_buckets: Buckets whose versioning is enabled. Their ``null`` versions are distinct from their unversioned keys. Use the same collection for :meth:`conflict_candidates`. @@ -211,12 +272,12 @@ def move_pairs( Also if ``versioning_enabled_buckets`` is a string. """ named = self._check(pairs, missing, versioning_enabled_buckets, "move") - return [(p1, p2) for p1, p2, _, source, dest in named if source != dest] + return [(pair.path1, pair.path2) for pair in named if pair.source != pair.destination] def check_copy_pairs( self, - pairs: Sequence[tuple[str, str]], - missing: Collection[str] | None = None, + pairs: Sequence[tuple[str | S3Path, str | S3Path]], + missing: Collection[str | S3Path] | None = None, *, versioning_enabled_buckets: Collection[str] = (), ) -> None: @@ -230,10 +291,11 @@ def check_copy_pairs( Args: pairs: The sources and destinations of the copy, as - :meth:`copy_pairs` of this pairing returns them. + :meth:`copy_pairs` or :meth:`s3_copy_pairs` of this pairing + returns them. missing: The :meth:`conflict_candidates` without an object at their - key, in any form that names them; needed when there are - candidates. + key, as strings in any form that names them or as ``S3Path`` + objects; needed when there are candidates. versioning_enabled_buckets: Buckets whose versioning is enabled. Use the same collection for :meth:`conflict_candidates`. @@ -249,11 +311,11 @@ def check_copy_pairs( def _check( self, - pairs: Sequence[tuple[str, str]], - missing: Collection[str] | None, + pairs: Sequence[tuple[str | S3Path, str | S3Path]], + missing: Collection[str | S3Path] | None, versioning_enabled_buckets: Collection[str], operation: str, - ) -> list[tuple[str, str, bool, str, str]]: + ) -> list[_NamedPair]: """Raise for the conflicts of a move or a copy. Args: @@ -279,15 +341,15 @@ def _check( if missing is None and candidates: raise ValueError("missing is needed to check the pairs.") # A directory without an object at its key writes no destination. - skipped = set(candidates).intersection( - self._target(S3Path.parse(path), versioning_enabled_buckets) for path in missing or () + skipped = {pair.source for pair in candidates}.intersection( + self._target(S3Path.of(path), versioning_enabled_buckets) for path in missing or () ) # A path with a version always names an object, so it writes its # destination even when the key has no current object. writers = Counter( - dest - for _, _, versioned, source, dest in named - if source != dest and (versioned or source not in skipped) + pair.destination + for pair in named + if pair.source != pair.destination and (pair.versioned or pair.source not in skipped) ) for dest in writers: if writers[dest] > 1: @@ -337,7 +399,7 @@ def _is_glob(path: str) -> bool: return has_magic(path) and not S3Path.has_version_id(path) @staticmethod - def _target(path: S3Path, versioning_enabled_buckets: Collection[str]) -> str: + def _target(path: S3Path, versioning_enabled_buckets: Collection[str]) -> S3Path: """Return a move target using the caller's bucket versioning state. Args: @@ -345,17 +407,17 @@ def _target(path: S3Path, versioning_enabled_buckets: Collection[str]) -> str: versioning_enabled_buckets: Buckets whose versioning is enabled. Returns: - The normalized path, retaining a ``null`` version only when its - bucket has versioning enabled. + The path, retaining a ``null`` version only when its bucket has + versioning enabled. """ - return str(path if path.bucket in versioning_enabled_buckets else path.target) + return path if path.bucket in versioning_enabled_buckets else path.target @staticmethod def _moves( - pairs: Sequence[tuple[str, str]], + pairs: Sequence[tuple[str | S3Path, str | S3Path]], versioning_enabled_buckets: Collection[str] = (), copy: bool = False, - ) -> tuple[list[tuple[str, str, bool, str, str]], set[str], list[str]]: + ) -> tuple[list[_NamedPair], set[S3Path], list[_NamedPair]]: """Compare the paths of a move by what they name. Args: @@ -366,8 +428,8 @@ def _moves( Returns: Each pair with whether its source has a version and the targets of its source and destination; the targets of all sources, - including those left in place; and the conflict candidates (see - :meth:`conflict_candidates`). + including those left in place; and the pairs of the conflict + candidates (see :meth:`conflict_candidates`). Raises: TypeError: If ``versioning_enabled_buckets`` is a string. @@ -378,32 +440,52 @@ def _moves( ) named = [] for p1, p2 in pairs: - source_path = S3Path.parse(p1) + source_path = S3Path.of(p1) named.append( - ( + _NamedPair( p1, p2, bool(source_path.version_id), S3PathPairing._target(source_path, versioning_enabled_buckets), - S3PathPairing._target(S3Path.parse(p2), versioning_enabled_buckets), + S3PathPairing._target(S3Path.of(p2), versioning_enabled_buckets), ) ) # The sources left in place count too; a copy onto one overwrites it. - sources = {source for _, _, _, source, _ in named} - # A source with another source below it may be a directory. + sources = {pair.source for pair in named} + # A source with another source below it may be a directory, named + # as it is listed: the parents of a name are cut at its last slash. directories: set[str] = set() for source in sources: - parent = source.rpartition("/")[0] + parent = source.name.rpartition("/")[0] while parent and parent not in directories: directories.add(parent) parent = parent.rpartition("/")[0] - counts = Counter(dest for _, _, _, source, dest in named if source != dest) + counts = Counter(pair.destination for pair in named if pair.source != pair.destination) candidates = [ - source - for _, _, versioned, source, dest in named - if source != dest - and (counts[dest] > 1 or dest in sources) - and (copy or source in directories) - and not versioned + pair + for pair in named + if pair.source != pair.destination + and (counts[pair.destination] > 1 or pair.destination in sources) + and (copy or pair.source.name in directories) + and not pair.versioned ] return named, sources, candidates + + +@dataclass(frozen=True) +class _NamedPair: + """A pair of a move or a copy with the paths that it names. + + Attributes: + path1: The source, as given in the pairs. + path2: The destination, as given in the pairs. + versioned: Whether the source has a version. + source: The target of the source (see ``S3PathPairing._target``). + destination: The target of the destination. + """ + + path1: str | S3Path + path2: str | S3Path + versioned: bool + source: S3Path + destination: S3Path diff --git a/tests/pyathena/filesystem/test_s3.py b/tests/pyathena/filesystem/test_s3.py index ee9da2f9b..0a729bb60 100644 --- a/tests/pyathena/filesystem/test_s3.py +++ b/tests/pyathena/filesystem/test_s3.py @@ -68,6 +68,7 @@ MULTIPART_COPY_BLOCK_SIZE, MULTIPART_COPY_KWARGS, MULTIPART_COPY_SIZE, + BranchRecordingCallback, OpaquePath, ThrottledOperation, forking, @@ -77,6 +78,7 @@ poll_until, retried, run_in_forked_child, + s3_copy_targets, s3_request_targets, serve_s3_keys, shutdown_bounded, @@ -1256,6 +1258,117 @@ def test_fan_outs_act_on_listed_keys(self, call, result, sent): assert call(fs) == result assert s3_request_targets(requests) == sent + # The key "a/x" and a key that a string path would name as its null + # version, which a move compares with the key itself. + NULL_KEYS = ("a/x", "a/x?versionId=null") + + @pytest.mark.parametrize( + ("call", "stored", "copied", "deleted"), + [ + pytest.param( + lambda fs: fs.copy("s3://bucket/a", "s3://bucket/b", recursive=True), + {"a/x", "a/x?versionId=null", "b/x", "b/x?versionId=null"}, + [("a/x", None, "b/x"), ("a/x?versionId=null", None, "b/x?versionId=null")], + [], + id="copy", + ), + pytest.param( + lambda fs: fs.copy("s3://bucket/a/*", "s3://bucket/c/"), + {"a/x", "a/x?versionId=null", "c/x", "c/x?versionId=null"}, + [("a/x", None, "c/x"), ("a/x?versionId=null", None, "c/x?versionId=null")], + [], + id="copy-glob", + ), + pytest.param( + lambda fs: fs.mv("s3://bucket/a", "s3://bucket/b", recursive=True), + {"b/x", "b/x?versionId=null"}, + [("a/x", None, "b/x"), ("a/x?versionId=null", None, "b/x?versionId=null")], + [("delete_objects", "a/x", None), ("delete_objects", "a/x?versionId=null", None)], + id="mv", + ), + ], + ) + def test_copies_act_on_listed_keys(self, call, stored, copied, deleted): + # GH-1403: copy() and mv() copy each key that a listing returns to a + # destination named after the key as it is, also a key that ends in + # what looks like a version ID query, never the version that parsing + # its name would give. + fs = S3FileSystem( + key="dummy", secret="dummy", region_name="us-east-1", skip_instance_cache=True + ) + store, requests = serve_s3_keys(fs, self.NULL_KEYS) + call(fs) + assert store == stored + assert s3_copy_targets(requests) == copied + assert [t for t in s3_request_targets(requests) if t[0] == "delete_objects"] == deleted + + def test_get_downloads_listed_keys(self, tmp_path): + # GH-1403: get() downloads each key that a listing returns by its + # bucket and key to a local path named after the key as it is, and + # names the branch of the callback after the key as a string. + fs = S3FileSystem( + key="dummy", secret="dummy", region_name="us-east-1", skip_instance_cache=True + ) + _, requests = serve_s3_keys(fs, self.NULL_KEYS) + callback = BranchRecordingCallback() + fs.get("s3://bucket/a", str(tmp_path / "out"), recursive=True, callback=callback) + assert sorted(os.listdir(tmp_path / "out")) == ["x", "x?versionId=null"] + assert [t for t in s3_request_targets(requests) if t[0] == "get_object"] == [ + ("get_object", "a/x", None), + ("get_object", "a/x?versionId=null", None), + ] + assert sorted(callback.sources) == ["bucket/a", "bucket/a/x", "bucket/a/x?versionId=null"] + + @pytest.mark.parametrize( + ("path", "version_id", "sent"), + [ + ( + S3Path("bucket", "a/x?versionId=null"), + None, + ("head_object", "a/x?versionId=null", None), + ), + ( + S3Path("bucket", "a/x?versionId=null"), + "v2", + ("head_object", "a/x?versionId=null", "v2"), + ), + ], + ) + def test_info_looks_up_path_as_it_is(self, path, version_id, sent): + # GH-1403: info() looks an S3Path up by its bucket, key and version, + # as given or from version_id, never by parsing its name. + fs = S3FileSystem( + key="dummy", secret="dummy", region_name="us-east-1", skip_instance_cache=True + ) + _, requests = serve_s3_keys(fs, self.NULL_KEYS) + assert fs.info(path, version_id=version_id)["name"] == "bucket/a/x?versionId=null" + assert s3_request_targets(requests) == [sent] + + @pytest.mark.parametrize( + ("call", "sent"), + [ + pytest.param( + lambda fs: fs.copy(["s3://bucket/a/x", "s3://bucket/a/*"], "s3://bucket/b/"), + [("copy_object", "b/x", None)], + id="copy", + ), + pytest.param( + lambda fs: fs.rm(["s3://bucket/a/x", "s3://bucket/a/*"]), + [("delete_objects", "a/x", None)], + id="rm", + ), + ], + ) + def test_expansion_acts_on_an_object_once(self, call, sent): + # GH-1403: a path that the caller passes and a listed key that name + # the same object are acted on once, as expand_path() names them once. + fs = S3FileSystem( + key="dummy", secret="dummy", region_name="us-east-1", skip_instance_cache=True + ) + _, requests = serve_s3_keys(fs, {"a/x"}) + call(fs) + assert [t for t in s3_request_targets(requests) if t[0] != "head_object"] == sent + def test_expand_path_expands_below_listed_matches(self): # GH-1402: a recursive glob expands below each match by its key prefix. fs = S3FileSystem( @@ -2231,7 +2344,9 @@ def test_mv_conflicting_destinations(self, path1, path2): def test_mv_conflicting_destinations_of_listed_sources(self): # A list of sources is mapped without looking up the destination. fs = self._make_fs() - fs.expand_path = mock.MagicMock(return_value=["bucket/x/a", "bucket/y/a"]) + fs._expand_paths = mock.MagicMock( + return_value=[(p, S3Path.parse(p)) for p in ("bucket/x/a", "bucket/y/a")] + ) fs.isdir = mock.MagicMock() with pytest.raises(ValueError, match="same destination"): @@ -2707,7 +2822,9 @@ def test_freed_by_reference_counting(self): def test_mv_nothing_within_maxdepth(self): # Only directories within maxdepth: nothing is moved, as with copy(). fs = self._make_fs() - fs.expand_path = mock.MagicMock(return_value=["bucket/src/sub"]) + fs._expand_paths = mock.MagicMock( + return_value=[("bucket/src/sub", S3Path("bucket", "src/sub"))] + ) fs.isdir = mock.MagicMock(return_value=True) fs.mv("s3://bucket/src", "s3://bucket/dst/", recursive=True, maxdepth=1) @@ -3063,11 +3180,12 @@ def test_get_put_pass_kwargs_to_expansion(self, tmp_path): ) as expand_path: fs.put(str(tmp_path / "a"), "s3://bucket/dst/", assume_literal=True) assert expand_path.call_args.kwargs["assume_literal"] is True - fs.expand_path = mock.MagicMock(return_value=["bucket/root/a"]) + fs._expand_paths = mock.MagicMock( + return_value=[("bucket/root/a", S3Path("bucket", "root/a"))] + ) fs.get_file = mock.MagicMock() - fs.isdir = mock.MagicMock(return_value=False) fs.get("s3://bucket/root/", str(tmp_path / "out") + "/", recursive=True, prefix="a") - assert fs.expand_path.call_args.kwargs["prefix"] == "a" + assert fs._expand_paths.call_args.kwargs["prefix"] == "a" @pytest.mark.parametrize("on_error", ["return", "raise"]) def test_cat_ranges_on_error(self, on_error): diff --git a/tests/pyathena/filesystem/test_s3_async.py b/tests/pyathena/filesystem/test_s3_async.py index 23574a21b..fc733ed76 100644 --- a/tests/pyathena/filesystem/test_s3_async.py +++ b/tests/pyathena/filesystem/test_s3_async.py @@ -50,6 +50,7 @@ MOVE_SOURCE_PARAMS, MULTIPART_COPY_BLOCK_SIZE, MULTIPART_COPY_KWARGS, + BranchRecordingCallback, OpaquePath, ThrottledOperation, await_bounded, @@ -62,6 +63,7 @@ retried, run_bounded, run_in_forked_child, + s3_copy_targets, s3_request_targets, serve_s3_keys, shutdown_bounded, @@ -736,6 +738,64 @@ async def test_fan_outs_act_on_listed_keys(self, call, result, sent): assert await await_bounded(call(fs)) == result assert s3_request_targets(requests) == sent + @pytest.mark.parametrize( + ("call", "stored", "copied", "deleted"), + [ + pytest.param( + lambda fs: fs._copy("s3://bucket/a", "s3://bucket/b", recursive=True), + {"a/x", "a/x?versionId=null", "b/x", "b/x?versionId=null"}, + [("a/x", None, "b/x"), ("a/x?versionId=null", None, "b/x?versionId=null")], + [], + id="copy", + ), + pytest.param( + lambda fs: fs._copy("s3://bucket/a/*", "s3://bucket/c/"), + {"a/x", "a/x?versionId=null", "c/x", "c/x?versionId=null"}, + [("a/x", None, "c/x"), ("a/x?versionId=null", None, "c/x?versionId=null")], + [], + id="copy-glob", + ), + pytest.param( + lambda fs: fs._mv("s3://bucket/a", "s3://bucket/b", recursive=True), + {"b/x", "b/x?versionId=null"}, + [("a/x", None, "b/x"), ("a/x?versionId=null", None, "b/x?versionId=null")], + [("delete_objects", "a/x", None), ("delete_objects", "a/x?versionId=null", None)], + id="mv", + ), + ], + ) + @pytest.mark.asyncio + async def test_copies_act_on_listed_keys(self, call, stored, copied, deleted): + # GH-1403: the asynchronous copies act on the keys that a listing + # returns, as those of S3FileSystem do. + fs = AioS3FileSystem( + key="dummy", secret="dummy", region_name="us-east-1", skip_instance_cache=True + ) + store, requests = serve_s3_keys(fs._sync_fs, {"a/x", "a/x?versionId=null"}) + await await_bounded(call(fs)) + assert store == stored + assert s3_copy_targets(requests) == copied + assert [t for t in s3_request_targets(requests) if t[0] == "delete_objects"] == deleted + + @pytest.mark.asyncio + async def test_get_downloads_listed_keys(self, tmp_path): + # GH-1403: _get() downloads the keys that a listing returns as get() + # does, with the branches of the callback named after them as strings. + fs = AioS3FileSystem( + key="dummy", secret="dummy", region_name="us-east-1", skip_instance_cache=True + ) + _, requests = serve_s3_keys(fs._sync_fs, {"a/x", "a/x?versionId=null"}) + callback = BranchRecordingCallback() + await await_bounded( + fs._get("s3://bucket/a", str(tmp_path / "out"), recursive=True, callback=callback) + ) + assert sorted(os.listdir(tmp_path / "out")) == ["x", "x?versionId=null"] + assert [t for t in s3_request_targets(requests) if t[0] == "get_object"] == [ + ("get_object", "a/x", None), + ("get_object", "a/x?versionId=null", None), + ] + assert sorted(callback.sources) == ["bucket/a", "bucket/a/x", "bucket/a/x?versionId=null"] + @pytest.mark.asyncio async def test_du_cancellation_sends_no_further_lookup(self): # GH-1402: as fsspec's _du(), the lookups of the found entries are diff --git a/tests/pyathena/filesystem/test_s3_path.py b/tests/pyathena/filesystem/test_s3_path.py index 8280d9b22..c72749c67 100644 --- a/tests/pyathena/filesystem/test_s3_path.py +++ b/tests/pyathena/filesystem/test_s3_path.py @@ -55,6 +55,13 @@ class TestS3Path: def test_parse(self, path, expected): assert S3Path.parse(path) == expected + def test_of(self): + # GH-1403: a string is parsed, and an S3Path, such as a listed key + # that ends in what looks like a version ID query, is kept as it is. + listed = S3Path("bucket", "a/x?versionId=null") + assert S3Path.of(listed) is listed + assert S3Path.of("s3://bucket/a/x?versionId=null") == S3Path("bucket", "a/x", "null") + def test_empty_key_is_the_bucket(self): # GH-1401: an empty key names the bucket, which the path prints as. assert S3Path("bucket", "") == S3Path("bucket") diff --git a/tests/pyathena/filesystem/test_s3_path_pairing.py b/tests/pyathena/filesystem/test_s3_path_pairing.py index 6685a4dc2..933758c7e 100644 --- a/tests/pyathena/filesystem/test_s3_path_pairing.py +++ b/tests/pyathena/filesystem/test_s3_path_pairing.py @@ -7,6 +7,7 @@ import pytest +from pyathena.filesystem.s3_path import S3Path from pyathena.filesystem.s3_path_pairing import S3PathPairing @@ -125,6 +126,69 @@ def test_looks_up_destination(self, path1, path2, expected): def test_copy_pairs(self, path1, path2, sources, destination_is_dir, expected): assert S3PathPairing(path1, path2).copy_pairs(sources, destination_is_dir) == expected + def test_copy_pairs_names_path_sources_by_key(self): + # GH-1403: an S3Path source, such as a listed key, is named after its + # bucket and key as they are, while a source string with a version ID + # is named after its key. + listed = S3Path("bucket", "d/x?versionId=null") + pairing = S3PathPairing("s3://bucket/d", "out", recursive=True) + assert pairing.copy_pairs(["bucket/d", listed], False) == [ + ("bucket/d", "out"), + (listed, "out/x?versionId=null"), + ] + + @pytest.mark.parametrize( + ("path1", "path2", "sources", "destination_is_dir", "expected"), + [ + # The destination root parsed, with the suffix of the listed key. + ( + "s3://bucket/d", + "s3://bucket/out", + ["bucket/d", S3Path("bucket", "d/x?versionId=null")], + False, + [ + ("bucket/d", "s3://bucket/out"), + ( + S3Path("bucket", "d/x?versionId=null"), + S3Path("bucket", "out/x?versionId=null"), + ), + ], + ), + # A bucket root, given with a trailing slash. + ( + "s3://bucket/d/*", + "s3://other/", + [S3Path("bucket", "d/x?versionId=null")], + None, + [(S3Path("bucket", "d/x?versionId=null"), S3Path("other", "x?versionId=null"))], + ), + # A list of sources, copied into the destination by name. + ( + ["s3://bucket/d"], + "s3://bucket/out", + ["bucket/d", S3Path("bucket", "d/x?versionId=null")], + None, + [ + ("bucket/d", "s3://bucket/out/d"), + ( + S3Path("bucket", "d/x?versionId=null"), + S3Path("bucket", "out/x?versionId=null"), + ), + ], + ), + ], + ) + def test_s3_copy_pairs(self, path1, path2, sources, destination_is_dir, expected): + # GH-1403: the destination of an S3Path source is an S3Path, never a + # string that parsing would read as a version. + pairing = S3PathPairing(path1, path2, recursive=True) + assert pairing.s3_copy_pairs(sources, destination_is_dir) == expected + + def test_s3_copy_pairs_versioned_destination(self): + pairing = S3PathPairing("s3://bucket/d", "s3://bucket/out?versionId=v1", recursive=True) + with pytest.raises(ValueError, match="versioned"): + pairing.s3_copy_pairs([S3Path("bucket", "d/x")], False) + def test_copy_pairs_needs_destination_lookup(self): with pytest.raises(ValueError, match="destination_is_dir"): S3PathPairing("s3://bucket/a", "s3://bucket/b").copy_pairs(["bucket/a"]) @@ -214,6 +278,28 @@ def test_move_pairs_directory_without_object(self): # The missing sources can be given in any form that names them. assert _pairing(pairs).move_pairs(pairs, missing={"s3://bucket/d"}) == pairs + def test_move_pairs_keeps_listed_keys_apart_from_versions(self): + # GH-1403: the S3Path of a key that ends in a null version query is + # compared as that key, not as the null version, which is the key + # without the query. + pairs = [ + (S3Path("bucket", "a/x"), S3Path("bucket", "b/x")), + (S3Path("bucket", "a/x?versionId=null"), S3Path("bucket", "b/x?versionId=null")), + ] + assert _pairing(pairs).move_pairs(pairs) == pairs + + def test_move_pairs_path_conflict_candidates(self): + # GH-1403: the candidates of S3Path sources are S3Path objects, and + # missing ones may be given as such; their directories are named as + # they are listed. + directory = S3Path("bucket", "d?versionId=v1") + pairs = [ + (directory, S3Path("bucket", "e")), + (S3Path("bucket", "d?versionId=v1/x"), S3Path("bucket", "e")), + ] + assert _pairing(pairs).conflict_candidates(pairs) == [directory] + assert _pairing(pairs).move_pairs(pairs, missing={directory}) == pairs + def test_move_pairs_missing_is_a_collection(self): pairs = [("s3://bucket/d", "s3://bucket/e"), ("s3://bucket/d/x", "s3://bucket/e")] with pytest.raises(TypeError, match="collection"): diff --git a/tests/pyathena/util.py b/tests/pyathena/util.py index d39594b73..18e9de745 100644 --- a/tests/pyathena/util.py +++ b/tests/pyathena/util.py @@ -29,6 +29,7 @@ from botocore.config import Config from botocore.exceptions import ClientError from botocore.response import StreamingBody +from fsspec.callbacks import Callback from fsspec.implementations.dirfs import DirFileSystem from fsspec.implementations.local import LocalFileSystem from jinja2 import Environment, FileSystemLoader @@ -1812,6 +1813,39 @@ def __fspath__(self): return self._path +class BranchRecordingCallback(Callback): + """A progress callback that records the source paths of its branches.""" + + def __init__(self): + super().__init__() + self.sources = [] + + def branched(self, path_1, path_2, **kwargs): + self.sources.append(path_1) + return super().branched(path_1, path_2, **kwargs) + + +def s3_copy_targets(requests): + """Return the sources and destinations of recorded CopyObject requests. + + Args: + requests: The requests that :func:`serve_s3_keys` records. + + Returns: + The sorted tuples of the source key, the source version ID, and the + destination key. + """ + return sorted( + ( + request["CopySource"]["Key"], + request["CopySource"].get("VersionId"), + request["Key"], + ) + for name, request in requests + if name == "copy_object" + ) + + def s3_request_targets(requests): """Return the operations, keys and version IDs of recorded S3 requests.