diff --git a/docs/tutorials/sca/tiny_aes.py b/docs/tutorials/sca/tiny_aes.py index 1c57f3ae..f23cece7 100644 --- a/docs/tutorials/sca/tiny_aes.py +++ b/docs/tutorials/sca/tiny_aes.py @@ -33,7 +33,7 @@ from scaaml.models import get_gpam_model from sedpack.io import ( Dataset, - DatasetFillerContext, + DatasetFiller, Metadata, DatasetStructure, Attribute, @@ -41,7 +41,7 @@ from sedpack.io.types import SplitT -def add_shard(shard_file: Path, dataset_filler: DatasetFillerContext, +def add_shard(shard_file: Path, dataset_filler: DatasetFiller, split: SplitT) -> tuple[float, float]: # Reading these one by one is slow. shard = np.load(shard_file) diff --git a/project-words.txt b/project-words.txt index f9a9a043..20994953 100644 --- a/project-words.txt +++ b/project-words.txt @@ -46,6 +46,8 @@ compresslevel convnet crossentropy ddof +deepcopy +deque diutils dtype dtypes @@ -76,6 +78,8 @@ patchelf perfcounters perfetto pickleable +popitem +popleft pyarray pyclass pyfunction diff --git a/src/sedpack/io/__init__.py b/src/sedpack/io/__init__.py index 79408551..a6b35937 100644 --- a/src/sedpack/io/__init__.py +++ b/src/sedpack/io/__init__.py @@ -14,7 +14,7 @@ """Dataset creation and usage.""" from sedpack.io.dataset import Dataset -from sedpack.io.dataset_filler import DatasetFiller, DatasetFillerContext +from sedpack.io.dataset_filler import DatasetFiller from sedpack.io.metadata import Attribute from sedpack.io.metadata import DatasetStructure from sedpack.io.metadata import Metadata @@ -23,7 +23,6 @@ "Attribute", "Dataset", "DatasetFiller", - "DatasetFillerContext", "DatasetStructure", "Metadata", ] diff --git a/src/sedpack/io/dataset_filler.py b/src/sedpack/io/dataset_filler.py index c858a35c..2700163f 100644 --- a/src/sedpack/io/dataset_filler.py +++ b/src/sedpack/io/dataset_filler.py @@ -15,15 +15,18 @@ number.""" from __future__ import annotations +from collections import OrderedDict, deque import concurrent.futures -import dataclasses +import copy +import json from pathlib import Path from types import TracebackType -from typing import Any, Type, TYPE_CHECKING +from typing import Any, Type, TYPE_CHECKING, get_args import tenacity from sedpack.io.file_info import PathGenerator, FileInfo +from sedpack.io.merge_shard_infos import merge_splits_shard_infos from sedpack.io.shard_file_metadata import ShardInfo, ShardsList, ShardListInfo from sedpack.io.shard import Shard from sedpack.io.types import ExampleT, SplitT @@ -37,6 +40,7 @@ @tenacity.retry( wait=tenacity.wait_random(min=60, max=5 * 60), # 1 to 5 minutes stop=tenacity.stop_after_attempt(10), + retry=tenacity.retry_if_exception_type(OSError), reraise=True, ) def _close_shard(shard: Shard) -> ShardInfo: @@ -46,111 +50,172 @@ def _close_shard(shard: Shard) -> ShardInfo: return shard.close() -@dataclasses.dataclass -class ShardProgress: - """Internal information about a shard progress. Since we do not want to - close a shard each time an example is written into another split we keep - multiple shards open. This makes keeping information easier. - """ - shard: Shard - written_examples: int = 0 - - -class DatasetFillerContext: - """The actual context. Use DatasetFiller which closes the last shard on - exit. Takes care of creating new shards with proper part_number and +class DatasetFiller: + """Takes care of creating new shards with the correct part_number and group_number. + + Main use-cases: + 1. Standard single-coordinator writing (prefer `dataset.filler()`, or + `DatasetFiller(dataset=dataset)` with default + `relative_path_from_split=Path(".")`): + When `relative_path_from_split` is empty (`Path(".")` or `Path("")`), + exiting the `with` block writes all shard metadata files and updates all + the way up to `/shards_list.json` and the root + `dataset_info.json`. + ```python + # Standalone use, prefer using sedpack.io.DatasetWriting.filler instead. + # Context manager properly opens and closes shards. + with DatasetFiller( + dataset=dataset, + ) as dataset_filler: + # Add examples, new shards are opened automatically. + for values in examples: + dataset_filler.write_example(values=values, split=TRAIN_SPLIT) + ``` + 2. Built-in multi-process writing: + Prefer `dataset.write_multiprocessing(...)`, which automatically assigns + each worker a unique non-empty `relative_path_from_split` and merges + all worker updates into `dataset_info.json` once the workers finish. + + Advanced use-case (custom multi-worker / distributed writing): + When running multiple `DatasetFiller` instances across custom worker + processes or distributed jobs (without `Dataset.write_multiprocessing`), + pass a unique, non-empty `relative_path_from_split` (e.g., + `Path(f"worker_{worker_id}")`) to each worker's `DatasetFiller`. Because + `relative_path_from_split` is non-empty, exiting the `with` block writes + and hashes the worker's shard lists only up to + `//shards_list.json`, without touching + `/shards_list.json` or the root `dataset_info.json`. Once all + workers finish, collect `get_updated_infos()` on the coordinator and call + `dataset.write_config(updated_infos=...)`: + ```python + def run_worker( + dataset_path: Path, + worker_id: int, + items: list[ExampleT], + ) -> list[ShardListInfo]: + ds = Dataset(dataset_path) + filler = DatasetFiller( + dataset=ds, + relative_path_from_split=Path(f"worker_{worker_id}"), + ) + with filler: + for item in items: + filler.write_example(values=item, split=TRAIN_SPLIT) + return filler.get_updated_infos() + + # Coordinator: merge all worker ShardListInfo updates into the dataset. + updated_infos: list[ShardListInfo] = [] + for worker_id, chunk in enumerate(chunks): + updated_infos.extend(run_worker(dataset.path, worker_id, chunk)) + dataset.write_config(updated_infos=updated_infos) + ``` + + Caveats for the advanced use-case: + - `relative_path_from_split` MUST be non-empty for every concurrent worker + even when each worker writes to a different split (e.g., one worker + writing only `"train"` and another writing only `"test"`). Leaving + `relative_path_from_split` empty (`Path(".")`) causes `__exit__` to call + `dataset.write_config(...)` and overwrite the shared root + `dataset_info.json`, racing with other workers and losing their split + updates. + - Concurrent workers must use disjoint `relative_path_from_split` paths + where neither path is a parent prefix of another (e.g., do not run + `Path("a")` and `Path("a/b")` concurrently, because the `Path("a")` + worker writes `/a/shards_list.json` on exit). Sibling paths such + as `Path("job_1/worker_0")` and `Path("job_1/worker_1")` are safe. + - When `relative_path_from_split` is non-empty, written shards are not + registered in `/shards_list.json` or `dataset_info.json` until the + coordinator calls `dataset.write_config(updated_infos=...)` with the + `ShardListInfo` objects returned by `filler.get_updated_infos()`. """ def __init__( - self, - dataset_root_path: Path, - dataset_structure: DatasetStructure, - relative_path_from_split: Path, - *, - concurrent_pool: concurrent.futures.Executor, - write_updates: bool = True, + self, + dataset: DatasetWriting, + concurrency: int = 4, + relative_path_from_split: Path = Path("."), ) -> None: - """Initialize a dataset filler context which writes examples and - automatically opens and closes shards (except possibly not closing the - last shard which is closed by `DatasetFiller`). + """Context manager for writing examples into a dataset. Args: - dataset_root_path (Path): Dataset root directory. + dataset (DatasetWriting): Dataset object representing the dataset we + are filling. - dataset_structure (DatasetStructure): What information is saved about - each example and how they are saved. + concurrency (int): Setting to a positive integer allows writing shard + files in parallel using a `ProcessPoolExecutor`. Defaults to 4. relative_path_from_split (Path): New shards are created inside - `dataset_root_path / split / relative_path_from_split`. May not - contain "..". - - concurrent_pool (concurrent.futures.Executor): Shard file writing. - - write_updates (bool): Whether to save progress of written shard info - files. Defaults to `True`. In theory when set to `False` writing - could be faster (not writing the info file after each shard). But - that way progress may be lost (e.g., when the process is interrupted - during writing a shard file). + `dataset_root_path / split / relative_path_from_split` or children. + It will be created if it does not exist yet. When empty (`Path(".")`, + the default), `__exit__` updates metadata all the way up to + `/shards_list.json` and `dataset_info.json`. When non-empty + (e.g., `Path("worker_0")`), `__exit__` updates metadata only up to + `//shards_list.json` so multiple + workers can run without race conditions; the caller must then pass + `get_updated_infos()` to `dataset.write_config(updated_infos=...)` + (even if each worker writes to a different split). May not contain + "..". """ # Make sure that we do not accidentally traverse the directory above # `dataset_root_path`. self._relative_path_from_split: Path = FileInfo.no_directory_traversal( relative_path_from_split) - self._dataset_root_path: Path = dataset_root_path - self._dataset_structure: DatasetStructure = dataset_structure - self._concurrent_pool: concurrent.futures.Executor = concurrent_pool - self._write_updates: bool = write_updates + self._dataset: DatasetWriting = dataset + self._dataset_root_path: Path = dataset.path + self._dataset_structure: DatasetStructure = dataset.dataset_structure + + self._max_workers: int = max(1, concurrency) + self._concurrent_pool: concurrent.futures.Executor | None = None # Constants. - self._examples_per_shard: int = dataset_structure.examples_per_shard + self._examples_per_shard: int = ( + self._dataset_structure.examples_per_shard) - # Current shard. - self._current_shards_progress: dict[SplitT, ShardProgress] = {} + # LRU cache of currently open shards keyed by `(split, metadata_key)`. + # Bounded to at most `self._max_workers` open shards simultaneously. + self._current_shards: OrderedDict[tuple[SplitT, str], + Shard] = OrderedDict() - # Cumulated shard infos. - self._shards_lists: dict[SplitT, ShardsList] = {} - self._future_shard_infos: dict[ - SplitT, list[concurrent.futures.Future[ShardInfo]]] = {} + # Cumulated shard lists keyed by relative path to `shards_list.json`. + self._shards_lists: dict[Path, ShardsList] = {} + self._future_shard_infos: deque[ + concurrent.futures.Future[ShardInfo]] = deque() + self._updated_infos_by_split: dict[SplitT, ShardListInfo] = {} # Random path generator. self._path_generator = PathGenerator() + def _ensure_pool(self) -> concurrent.futures.Executor: + if self._concurrent_pool is None: + self._concurrent_pool = concurrent.futures.ProcessPoolExecutor( + max_workers=self._max_workers) + return self._concurrent_pool + + def _record_shard_info(self, shard_info: ShardInfo) -> None: + """Record a completed `ShardInfo` into `self._shards_lists`.""" + if shard_info.number_of_examples <= 0: + return + relative_path_self: Path = (shard_info.file_infos[0].file_path.parent / + "shards_list.json") + if relative_path_self not in self._shards_lists: + self._shards_lists[relative_path_self] = ShardsList.load_or_create( + dataset_root_path=self._dataset_root_path, + relative_path_self=relative_path_self, + ) + shards_list = self._shards_lists[relative_path_self] + shards_list.shard_files.append(shard_info) + shards_list.number_of_examples += shard_info.number_of_examples + @property - def shard_lists(self) -> dict[SplitT, ShardsList]: - """Return information about all shards written by this - DatasetFillerContext. + def shard_lists(self) -> dict[Path, ShardsList]: + """Return information about pending leaf shard lists buffered in this + `DatasetFiller` before `_update_infos()` flushes them on `__exit__`. """ - if not self._future_shard_infos: - return self._shards_lists - - for split, shard_info_futures in self._future_shard_infos.items(): - for future_info in shard_info_futures: - shard_info: ShardInfo = future_info.result() - - # Remember this shard info. - if split not in self._shards_lists: - # Load if exists. - self._shards_lists[split] = ShardsList.load_or_create( - dataset_root_path=self._dataset_root_path, - relative_path_self=shard_info.file_infos[0].file_path. - parent / "shards_list.json", - ) - self._shards_lists[split].shard_files.append(shard_info) - self._shards_lists[ - split].number_of_examples += shard_info.number_of_examples - - # Write down which shard files have been saved. - if self._write_updates: - for shards_list in self._shards_lists.values(): - shards_list.write_config( - dataset_root_path=self._dataset_root_path, - hashes=(), # We forget these now. - ) - - self._future_shard_infos = {} + while self._future_shard_infos: + self._record_shard_info(self._future_shard_infos.popleft().result()) return self._shards_lists def _get_new_shard(self, split: SplitT) -> Shard: @@ -170,148 +235,99 @@ def _get_new_shard(self, split: SplitT) -> Shard: dataset_root_path=self._dataset_root_path, ) - def write_example(self, - values: ExampleT, - split: SplitT, - custom_metadata: dict[str, Any] | None = None) -> None: + def write_example( + self, + values: ExampleT, + split: SplitT, + custom_metadata: dict[str, Any] | None = None, + ) -> None: """Write an example. Opens a new shard if necessary. + Up to `concurrency` (`max(1, concurrency)`) shards are kept open + concurrently across `(split, custom_metadata)` pairs using an LRU + policy, so oscillating `custom_metadata` or `split` values reuse open + shards instead of creating single-example shards. Examples without + `custom_metadata` (`None` or `{}`) and examples with non-empty + `custom_metadata` are always saved in separate shards. + Args: values (ExampleT): A dictionary with all values. split (SplitT): Which split to write this example into. - custom_metadata (dict[str, Any] | None): Optional metadata saved - with in the shard info. The shards then can be filtered using these - metadata. When a value is changed a new shard is open and the - current shard is closed. TODO there is no check if a shard with the - same `custom_metadata` already exists. When a `custom_metadata` is - provided for the very first time it is set for the current shard - (retroactively also affecting the previous writes). This behavior - is not ideal but a proper solution will require a proper design - (e.g., to avoid having too many open shards, finding out how to - find the shard with correct `custom_metadata` since a dictionary - cannot be easily hashed, etc.). + custom_metadata (dict[str, Any] | None): Optional JSON-serializable + metadata saved in the shard's `ShardInfo`. Shards can later be + filtered using these metadata. """ - # Check that we have a shard for the split. - if split not in self._current_shards_progress: - self._current_shards_progress[split] = ShardProgress( - self._get_new_shard(split=split)) - - current_progress: ShardProgress = self._current_shards_progress[split] - - # New metadata - previous_metadata = current_progress.shard.shard_info.custom_metadata - metadata_changed: bool = all(( - custom_metadata, - previous_metadata, - custom_metadata != previous_metadata, - )) - - # Open a new shard if the current one already contains too many - # examples. - if (current_progress.written_examples >= self._examples_per_shard or - metadata_changed): - # Close the current shard if needed. - self.close_shard(shard=current_progress.shard, split=split) - current_progress.shard = self._get_new_shard(split=split) - current_progress.written_examples = 0 - - # Update custom_metadata is needed + if split not in get_args(SplitT): + raise ValueError(f"Not a known split value: {split}") + + metadata_key: str = (json.dumps(custom_metadata, sort_keys=True) + if custom_metadata else "{}") + shard_key: tuple[SplitT, str] = (split, metadata_key) + + current_shard: Shard | None = self._current_shards.get(shard_key) + if current_shard is not None: + current_shard.write(values=values) + if (current_shard.shard_info.number_of_examples + >= self._examples_per_shard): + self.close_shard(shard=self._current_shards.pop(shard_key)) + else: + self._current_shards.move_to_end(shard_key) + return + + if len(self._current_shards) >= self._max_workers: + _, evicted_shard = self._current_shards.popitem(last=False) + self.close_shard(shard=evicted_shard) + + current_shard = self._get_new_shard(split=split) if custom_metadata: - current_progress.shard.shard_info.custom_metadata = custom_metadata - - # Write the current example and update counters. - current_progress.shard.write(values=values) - current_progress.written_examples += 1 - - # We have updated the current progress. - assert self._current_shards_progress[split] == current_progress - - def close_shard(self, shard: Shard, split: SplitT) -> None: - """Close shard. Called automatically by DatasetFiller.__exit__.""" - # Finish writing the shard. - if split not in self._future_shard_infos: - self._future_shard_infos[split] = [] - self._future_shard_infos[split].append( - self._concurrent_pool.submit( - _close_shard, - shard=shard, - )) + current_shard.shard_info.custom_metadata = copy.deepcopy( + custom_metadata) + current_shard.write(values=values) -class DatasetFiller: - """Takes care of creating new shards with the correct part_number and - group_number. - - Example use: - # Standalone use, prefer using sedpack.io.DatasetWriting.filler instead. - # Context manager properly opens and closes shards. - with DatasetFiller( - dataset=dataset, - ) as dataset_filler: - # Add examples, new shards are opened automatically. - for values in examples: - dataset_filler.write_example(values=values) - """ - - def __init__(self, - dataset: DatasetWriting, - concurrency: int = 1, - relative_path_from_split: Path = Path("."), - auto_update_dataset: bool = True) -> None: - """Context manager for writing examples into a dataset. - - Args: - - dataset (DatasetWriting): Dataset object representing the dataset we - are filling. - - concurrency (int): Setting to a positive integer allows writing shard - files in parallel. Defaults to 1 (sequential writes in another thread - or process). Thread is used for "tfrec" otherwise process. + if (current_shard.shard_info.number_of_examples + >= self._examples_per_shard): + self.close_shard(shard=current_shard) + else: + self._current_shards[shard_key] = current_shard - relative_path_from_split (Path): New shards are created inside - `dataset_root_path / split / relative_path_from_split` or children. - It will be created if it does not exist yet. Useful for - multiprocessing where we want to save changes without overwriting - files and race conditions. - - auto_update_dataset (bool): Defaults to `True`. When closed - (`__exit__` is called) write information about all written shards - back to the dataset. Defaults to `True`. When one creates multiple - `DatasetFiller` objects it is advised to set this parameter to - `False` since this is not thread safe. + def close_shard( + self, + shard: Shard, + split: SplitT | None = None, + ) -> None: + """Close shard asynchronously in the process pool. Called automatically + when a shard is full, evicted, or in `DatasetFiller.__exit__`. """ - self._concurrent_pool: concurrent.futures.Executor - if dataset.dataset_structure.shard_file_type == "tfrec": - self._concurrent_pool = concurrent.futures.ThreadPoolExecutor( - max_workers=1) - else: - self._concurrent_pool = concurrent.futures.ProcessPoolExecutor( - max_workers=max(1, concurrency),) - - self._dataset_filler_context: DatasetFillerContext - self._dataset_filler_context = DatasetFillerContext( - dataset_root_path=dataset.path, - dataset_structure=dataset.dataset_structure, - relative_path_from_split=relative_path_from_split, - concurrent_pool=self._concurrent_pool, - ) - self._auto_update_dataset: bool = auto_update_dataset - self._dataset: DatasetWriting = dataset - self._updated_infos: list[ShardListInfo] = [] + _ = split + if shard.shard_info.number_of_examples <= 0: + return + + while (len(self._future_shard_infos) >= self._max_workers or + (self._future_shard_infos and + self._future_shard_infos[0].done())): + self._record_shard_info(self._future_shard_infos.popleft().result()) + + self._future_shard_infos.append(self._ensure_pool().submit( + _close_shard, + shard=shard, + )) - def __enter__(self) -> DatasetFillerContext: - """Initialize DatasetFillerContext.""" - assert not self._updated_infos - return self._dataset_filler_context + def __enter__(self) -> DatasetFiller: + """Initialize the process pool for shard writing.""" + self._ensure_pool() + return self - def __exit__(self, exc_type: Type[BaseException] | None, - exc_value: BaseException | None, - exc_tb: TracebackType | None) -> None: - """Make sure to close the last shard. + def __exit__( + self, + exc_type: Type[BaseException] | None, + exc_value: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + """Make sure to close all open shards and update metadata files. Args: @@ -324,46 +340,64 @@ def __exit__(self, exc_type: Type[BaseException] | None, exc_tb (TracebackType | None): None if no exception, otherwise the traceback. """ - # Close the shard only if there was an example written. - split_progress = self._dataset_filler_context._current_shards_progress - for split, shard_progress in split_progress.items(): - if shard_progress.written_examples > 0: - # Need to do it this way when there would be a shard with less - # than examples_per_shard examples. - self._dataset_filler_context.close_shard( - shard=shard_progress.shard, - split=split, + try: + # Close each open shard that has at least one example written. + while self._current_shards: + _, shard = self._current_shards.popitem(last=False) + if shard.shard_info.number_of_examples > 0: + self.close_shard(shard=shard) + finally: + # Wait to finish writing of all files (after closing all shards but + # before updating metadata files), then release the pool so + # DatasetFiller is pickleable again. + if self._concurrent_pool is not None: + self._concurrent_pool.shutdown( + wait=True, + cancel_futures=False, ) - - # Wait to finish writing of all files (after closing all shards but - # before updating metadata files). - self._concurrent_pool.shutdown( - wait=True, - cancel_futures=False, - ) + self._concurrent_pool = None # Write updated info files. self._update_infos() - if self._auto_update_dataset: - # Note that when a DatasetFiller is pickled and unpickled it no - # longer has the same `self._dataset` object. - self._dataset.write_config(updated_infos=self._updated_infos) + if not self._relative_path_from_split.parts: + # When relative_path_from_split is empty, update all the way up to + # `/shards_list.json` and `dataset_info.json`. + self._dataset.write_config(updated_infos=self.get_updated_infos()) def _update_infos(self) -> None: - """Write back all information about written shards to the dataset. + """Write leaf `shards_list.json` files for all newly closed shards and + merge them up to `//shards_list.json`. """ - assert not self._updated_infos - # Update shards lists. - for shards_list in self._dataset_filler_context.shard_lists.values(): - self._updated_infos.append( - shards_list.write_config( - dataset_root_path=self._dataset.path, - hashes=self._dataset.dataset_structure. - hash_checksum_algorithms, - )) + while self._future_shard_infos: + self._record_shard_info(self._future_shard_infos.popleft().result()) + + if not self._shards_lists: + return + + # Write leaf shards_list.json files and clear the pending buffer so + # subsequent context manager runs only flush newly written shards. + leaf_infos: list[ShardListInfo] = [ + shards_list.write_config( + dataset_root_path=self._dataset_root_path, + hashes=self._dataset_structure.hash_checksum_algorithms, + ) for shards_list in self._shards_lists.values() + ] + self._shards_lists.clear() + + # Merge leaf shard lists up to + # `//shards_list.json`. + self._updated_infos_by_split.update( + merge_splits_shard_infos( + updates=leaf_infos, + dataset_root=self._dataset_root_path, + hashes=self._dataset_structure.hash_checksum_algorithms, + relative_path_from_split=self._relative_path_from_split, + )) def get_updated_infos(self) -> list[ShardListInfo]: - """Return list of all ShardListInfo to update the dataset. + """Return list of `ShardListInfo` objects to update the dataset (one + per written split pointing to + `//shards_list.json`). """ - return self._updated_infos + return list(self._updated_infos_by_split.values()) diff --git a/src/sedpack/io/dataset_writing.py b/src/sedpack/io/dataset_writing.py index 95b5e725..97fbbafa 100644 --- a/src/sedpack/io/dataset_writing.py +++ b/src/sedpack/io/dataset_writing.py @@ -12,16 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. """Mixin for sedpack.io.Dataset to do writing.""" -from collections import defaultdict +import concurrent.futures import contextlib import itertools -from multiprocessing import Pool from pathlib import Path from typing import ( Any, Callable, Mapping, - get_args, ) import uuid @@ -30,10 +28,9 @@ import sedpack from sedpack.io.dataset_base import DatasetBase from sedpack.io.file_info import FileInfo -from sedpack.io.merge_shard_infos import merge_shard_infos +from sedpack.io.merge_shard_infos import merge_splits_shard_infos from sedpack.io.shard_file_metadata import ShardsList, ShardListInfo -from sedpack.io.types import SplitT import sedpack.io.utils as diutils from sedpack.io.dataset_filler import DatasetFiller @@ -43,13 +40,14 @@ class DatasetWriting(DatasetBase): """ def write_multiprocessing( # pylint: disable=too-many-arguments - self, - *, - feed_writer: Callable[..., Any], - custom_arguments: list[Any], - custom_kwarguments: list[dict[str, Any]] | None = None, - consistency_check: bool = True, - single_process: bool = False) -> list[Any]: + self, + *, + feed_writer: Callable[..., Any], + custom_arguments: list[Any], + custom_kwarguments: list[dict[str, Any]] | None = None, + consistency_check: bool = True, + single_process: bool = False, + ) -> list[Any]: """Multiprocessing write support. Spawn `len(custom_arguments)` processes to write examples in parallel. Note that all computation is run on the CPU (using `tf.device("CPU")`) in order to prevent each @@ -82,7 +80,40 @@ def write_multiprocessing( # pylint: disable=too-many-arguments Returns: A list of values returned by calls to `feed_writer`. - Example use: See docs/tutorials/writing/ for more information. + Example use (see `docs/tutorials/writing/` for more information): + `feed_writer` takes `dataset_filler: DatasetFiller` as its first + positional argument, followed by arbitrary `*args` and `**kwargs` + unpacked from the worker's entry in `custom_arguments` and + `custom_kwarguments`: + ```python + def feed_worker( + dataset_filler: DatasetFiller, + chunk: list[ExampleT], + split: SplitT = TRAIN_SPLIT, + ) -> int: + # First argument is always `dataset_filler`; remaining `*args` and + # `**kwargs` come from `custom_arguments` and `custom_kwarguments`. + with dataset_filler: + for values in chunk: + dataset_filler.write_example( + values=values, + split=split, + ) + return len(chunk) + + counts = dataset.write_multiprocessing( + feed_writer=feed_worker, + custom_arguments=[(chunk_0,), (chunk_1,)], + custom_kwarguments=[{"split": TRAIN_SPLIT}, {"split": TEST_SPLIT}], + ) + ``` + Each worker receives a `DatasetFiller` isolated under a unique + non-empty `relative_path_from_split` (so exiting the worker's `with` + block updates metadata only up to + `//shards_list.json`); once all + workers finish, `write_multiprocessing` collects + `dataset_filler.get_updated_infos()` and calls + `self.write_config(updated_infos=updated_infos)`. """ # Default to no kwargs. if custom_kwarguments is None: @@ -100,7 +131,6 @@ def write_multiprocessing( # pylint: disable=too-many-arguments DatasetFiller( dataset=self, relative_path_from_split=Path(uuid.uuid4().hex), - auto_update_dataset=False, ) for _ in range(len(custom_arguments)) ] @@ -111,13 +141,17 @@ def write_multiprocessing( # pylint: disable=too-many-arguments custom_arguments, custom_kwarguments, ) - if single_process: + if single_process or not custom_arguments: # Testing mode. wrapper_outputs = list(map(_wrapper_func, wrapper_inputs)) else: - # Fill writers and collect results. - with Pool(len(custom_arguments)) as pool: - wrapper_outputs = list(pool.imap(_wrapper_func, wrapper_inputs)) + # Fill writers and collect results. Use ProcessPoolExecutor (whose + # workers are non-daemon) so each DatasetFiller can use its own + # ProcessPoolExecutor for shard writing. + with concurrent.futures.ProcessPoolExecutor( + max_workers=len(custom_arguments)) as executor: + wrapper_outputs = list( + executor.map(_wrapper_func, wrapper_inputs)) # Retrieve filled dataset_fillers and feed_writer results. dataset_fillers = [ @@ -140,13 +174,22 @@ def write_multiprocessing( # pylint: disable=too-many-arguments # Return user-defined results. return results - def filler(self, concurrency: int = 1) -> DatasetFiller: + def filler( + self, + concurrency: int = 4, + ) -> DatasetFiller: """Return a dataset filler context manager for writing examples. + Uses the default empty `relative_path_from_split=Path(".")`, so exiting + the `with` block automatically updates `/shards_list.json` and + the root `dataset_info.json`. For multi-worker or distributed writing, + use `Dataset.write_multiprocessing` or `DatasetFiller` with a non-empty + `relative_path_from_split` (see `DatasetFiller` docstring). + Args: concurrency (int): Setting to a positive integer allows writing shard - files in parallel. Defaults to 1 (sequential writes). + files in parallel. Defaults to 4. Example use: # Context manager properly opens and closes shards. @@ -158,7 +201,10 @@ def filler(self, concurrency: int = 1) -> DatasetFiller: split=split, ) """ - return DatasetFiller(dataset=self, concurrency=concurrency) + return DatasetFiller( + dataset=self, + concurrency=concurrency, + ) def write_config( self, @@ -166,41 +212,34 @@ def write_config( ) -> FileInfo: """Save configuration as json. + When multiple `DatasetFiller` instances are used with a non-empty + `relative_path_from_split` (see `DatasetFiller` docstring for the + advanced use-case and caveats), pass the concatenated + `filler.get_updated_infos()` list here after all fillers have exited. + Args: updated_infos (list[ShardListInfo] | None): List of updates or new - shard list infos. - - Returns: a tuple of hash checksums corresponding to the root dataset - info file. + `ShardListInfo` descriptors (as returned by + `DatasetFiller.get_updated_infos()`). Each `ShardListInfo` references + a written `shards_list.json` file (such as + `//shards_list.json` or a leaf + `shards_list.json`, with `file_path` relative to `dataset.path`) + along with its `hash_checksums`, `number_of_examples`, and + `number_of_shards`. `write_config` recursively merges these shard + lists up through intermediate directories to + `/shards_list.json` and writes the root `dataset_info.json`. + + Returns: a `FileInfo` with the path and hash checksums corresponding to + the root `dataset_info.json` file. """ - updated_infos_list: list[ShardListInfo] if updated_infos: - updated_infos_list = updated_infos - else: - updated_infos_list = [] - - # Update shards lists in splits. - splits_to_update: defaultdict[SplitT, - list[ShardListInfo]] = defaultdict(list) - - # Sort the infos by their split. - for info in updated_infos_list: - # The path is at least `split / shards_list.json` or longer. - split = str(info.shard_list_info_file.file_path.parts[0]) - # Type narrowing with get_args does not seem to work with mypy. - if split not in get_args(SplitT): - raise ValueError(f"Not a known split value: {split}") - splits_to_update[split].append(info) # type: ignore[index] - - # Merge recursively. - for split, updates in splits_to_update.items(): - self._dataset_info.splits[split] = merge_shard_infos( - updates=updates, - dataset_root=self.path, - common=1, - hashes=self.dataset_structure.hash_checksum_algorithms, - ) + self._dataset_info.splits.update( + merge_splits_shard_infos( + updates=updated_infos, + dataset_root=self.path, + hashes=self.dataset_structure.hash_checksum_algorithms, + )) # Update the main dataset info config. return diutils.safe_update_file( diff --git a/src/sedpack/io/merge_shard_infos.py b/src/sedpack/io/merge_shard_infos.py index 8fc49c32..8f43cb98 100644 --- a/src/sedpack/io/merge_shard_infos.py +++ b/src/sedpack/io/merge_shard_infos.py @@ -16,14 +16,18 @@ from collections import defaultdict from pathlib import Path +from typing import get_args from sedpack.io.shard_file_metadata import ShardsList, ShardListInfo -from sedpack.io.types import HashChecksumT +from sedpack.io.types import HashChecksumT, SplitT -def merge_shard_infos(updates: list[ShardListInfo], dataset_root: Path, - common: int, hashes: tuple[HashChecksumT, - ...]) -> ShardListInfo: +def merge_shard_infos( + updates: list[ShardListInfo], + dataset_root: Path, + common: int, + hashes: tuple[HashChecksumT, ...], +) -> ShardListInfo: """Merge a list of new `ShardListInfo`s into the dataset. Args: @@ -35,30 +39,35 @@ def merge_shard_infos(updates: list[ShardListInfo], dataset_root: Path, common (int): A positive integer indicating how many directories deep are shared between all `ShardListInfo`s (relative from the `dataset_root`). - When calling this function all `updates` have to be in the same split, - thus one should set `common=1`. It is not guaranteed to update - `shards_list.json` files all the way to the split when `common>1`. + All `updates` must share the first `common` path components (in + particular, the same split at index 0). Setting `common=1` merges all the + way up to `/shards_list.json`; setting + `common = 1 + len(relative_path_from_split.parts)` merges a worker's + subtree up to `//shards_list.json`. hashes (tuple[HashChecksumT, ...]): A tuple of hash checksum algorithms. """ - assert updates, "Nothing to update." + if not updates: + raise ValueError("Nothing to update.") + if common < 1: + raise ValueError(f"Expected common >= 1, got {common}") + + common_prefix = updates[0].shard_list_info_file.file_path.parts[:common] + if len(common_prefix) < common: + raise ValueError( + f"Update path {updates[0].shard_list_info_file.file_path} is " + f"shorter than common={common} directory components.") # Check that all common prefixes are the same. for update in updates: - if update.shard_list_info_file.file_path.parts[:common] != updates[ - 0].shard_list_info_file.file_path.parts[:common]: + parts = update.shard_list_info_file.file_path.parts + if len(parts) < common + 1: raise ValueError( - f"Not all relative paths are the same " - f"{update.shard_list_info_file.file_path.parts[:common]} vs " - f"{updates[0].shard_list_info_file.file_path.parts[:common]}") - - # The current level ShardsList (if it is in the updates). - root_shard_list: ShardsList = ShardsList.load_or_create( - dataset_root_path=dataset_root, - relative_path_self=Path().joinpath( - *updates[0].shard_list_info_file.file_path.parts[:common]) / - "shards_list.json", - ) + f"Update path {update.shard_list_info_file.file_path} must " + f"have at least {common + 1} path components.") + if parts[:common] != common_prefix: + raise ValueError(f"Not all relative paths are the same " + f"{parts[:common]} vs {common_prefix}") # Divide the updates to current level shard lists and the deeper ones. current_level: list[ShardListInfo] = [ @@ -69,49 +78,117 @@ def merge_shard_infos(updates: list[ShardListInfo], dataset_root: Path, update for update in updates if len(update.shard_list_info_file.file_path.parts) > common + 1 ] - # Check correctness of this implementation (O(1) check just to make sure we - # do not forget anything). assert len(current_level) + len(deeper_updates) == len(updates) - # Since the ShardsList is saved in a file named shards_list.json there can - # be at most one update in this depth. We can ignore it since it has been - # loaded into root_shard_list. - assert len(current_level) <= 1 - # Move children of root_shard_list into deeper_updates to let recursion - # merge everything. The updates are listed after the already present. - for child in root_shard_list.children_shard_lists: - root_shard_list.number_of_examples -= child.number_of_examples - deeper_updates = root_shard_list.children_shard_lists + deeper_updates - root_shard_list.children_shard_lists = [] + # If there are no deeper updates below this directory and the single + # ShardListInfo at this level already has the requested checksums computed, + # return it directly without re-traversing or re-hashing its subtree. + if (not deeper_updates and len(current_level) == 1 and + len(current_level[0].shard_list_info_file.hash_checksums) + == len(hashes)): + return current_level[0] - # Recursively update children with one longer common prefix. - # Sort the infos by common directory. + # The current level ShardsList on disk (already updated if current_level is + # non-empty). + root_shard_list: ShardsList = ShardsList.load_or_create( + dataset_root_path=dataset_root, + relative_path_self=Path().joinpath(*common_prefix) / "shards_list.json", + ) + + # Group deeper updates by their immediate child directory at depth `common`. recursively_update: defaultdict[str, list[ShardListInfo]] = defaultdict(list) for update in deeper_updates: - # The path is at least `split / $DIRECTORY / shards_list.json` or - # longer. current_path: Path = update.shard_list_info_file.file_path directory = str(current_path.parts[common]) recursively_update[directory].append(update) - # Recursively update. - merged: dict[str, ShardListInfo] = { # Merge recursively. + # If any existing child directory has no deeper updates but lacks the + # requested hash checksums, include it so its hashes are recomputed. + for child in root_shard_list.children_shard_lists: + child_dir = str(child.shard_list_info_file.file_path.parts[common]) + if (child_dir not in recursively_update and + len(child.shard_list_info_file.hash_checksums) != len(hashes)): + recursively_update[child_dir].append(child) + + # Recursively update only the child directories that changed. + merged: dict[str, ShardListInfo] = { directory: - merge_shard_infos(updates=recursive_updates, - dataset_root=dataset_root, - common=common + 1, - hashes=hashes) - for directory, recursive_updates in recursively_update.items() + merge_shard_infos( + updates=recursive_updates, + dataset_root=dataset_root, + common=common + 1, + hashes=hashes, + ) for directory, recursive_updates in recursively_update.items() } - # Merge the recursive into root_shard_list. - for child in merged.values(): - root_shard_list.number_of_examples += child.number_of_examples - root_shard_list.children_shard_lists.append(child) + # Preserve existing child directory ordering, replacing updated children in + # place and appending newly created child directories afterwards. + new_children: list[ShardListInfo] = [] + for child in root_shard_list.children_shard_lists: + child_dir = str(child.shard_list_info_file.file_path.parts[common]) + if child_dir in merged: + new_children.append(merged.pop(child_dir)) + else: + new_children.append(child) + new_children.extend(merged.values()) + root_shard_list.children_shard_lists = new_children + + root_shard_list.number_of_examples = ( + sum(s.number_of_examples for s in root_shard_list.shard_files) + + sum(c.number_of_examples for c in root_shard_list.children_shard_lists)) # Write the root_shard_list and return its ShardListInfo. return root_shard_list.write_config( dataset_root_path=dataset_root, hashes=hashes, ) + + +def merge_splits_shard_infos( + updates: list[ShardListInfo], + dataset_root: Path, + hashes: tuple[HashChecksumT, ...], + relative_path_from_split: Path = Path("."), +) -> dict[SplitT, ShardListInfo]: + """Group `updates` by split and merge each split up to + `//shards_list.json`. + + Args: + + updates (list[ShardListInfo]): ShardListInfo updates across one or more + splits. + + dataset_root (Path): Root directory of the dataset. + + hashes (tuple[HashChecksumT, ...]): Hash checksum algorithms to compute. + + relative_path_from_split (Path): Subdirectory relative to `` up to + which shard lists should be merged. When empty (`Path(".")`), merges all + the way up to `/shards_list.json`. + + Returns: Mapping from each updated `SplitT` to its merged `ShardListInfo`. + """ + splits_to_update: defaultdict[SplitT, + list[ShardListInfo]] = defaultdict(list) + for info in updates: + parts = info.shard_list_info_file.file_path.parts + if len(parts) < 2: + raise ValueError( + f"Expected ShardListInfo path inside a split directory, got " + f"{info.shard_list_info_file.file_path}") + split = str(parts[0]) + if split not in get_args(SplitT): + raise ValueError(f"Not a known split value: {split}") + splits_to_update[split].append(info) # type: ignore[index] + + common: int = 1 + len(relative_path_from_split.parts) + return { + split: + merge_shard_infos( + updates=split_updates, + dataset_root=dataset_root, + common=common, + hashes=hashes, + ) for split, split_updates in splits_to_update.items() + } diff --git a/tests/io/test_shard_custom_metadata.py b/tests/io/test_shard_custom_metadata.py index 7e45ec2c..431ac3ad 100644 --- a/tests/io/test_shard_custom_metadata.py +++ b/tests/io/test_shard_custom_metadata.py @@ -13,13 +13,14 @@ # limitations under the License. from pathlib import Path -from typing import Union +from typing import Any, Union import numpy as np import sedpack from sedpack.io import Dataset from sedpack.io import Metadata +from sedpack.io.shard_file_metadata import ShardInfo from sedpack.io.types import TRAIN_SPLIT @@ -60,42 +61,43 @@ def test_custom_shard_metadata(tmpdir: Union[str, Path]) -> None: custom_metadata_1 = {"key": {"key2": "valueB"}} with dataset.filler() as filler: - # No custom metadata. + # No custom metadata -> saved into its own shard (not overwritten by + # subsequent custom_metadata writes). filler.write_example( values={"attribute_name": array_of_values[0]}, split=TRAIN_SPLIT, ) - # Still the same shard, retroactively setting metadata here. + # Different custom_metadata -> opens a separate shard. filler.write_example( values={"attribute_name": array_of_values[1]}, split=TRAIN_SPLIT, custom_metadata=custom_metadata_0, ) - # Another shard has been open. + # Another shard opened for custom_metadata_1. filler.write_example( values={"attribute_name": array_of_values[2]}, split=TRAIN_SPLIT, custom_metadata=custom_metadata_1, ) - # Still the same shard. + # Still the same custom_metadata_1 shard. filler.write_example( values={"attribute_name": array_of_values[3]}, split=TRAIN_SPLIT, custom_metadata=custom_metadata_1, ) - # Still the same shard. + # Still the same custom_metadata_1 shard. filler.write_example( values={"attribute_name": array_of_values[4]}, split=TRAIN_SPLIT, custom_metadata=custom_metadata_1, ) - # Still the same shard. + # Reaches examples_per_shard=4 -> shard is closed immediately. filler.write_example( values={"attribute_name": array_of_values[5]}, split=TRAIN_SPLIT, custom_metadata=custom_metadata_1, ) - # Shard full, another opened. + # Opens a second shard for custom_metadata_1. filler.write_example( values={"attribute_name": array_of_values[6]}, split=TRAIN_SPLIT, @@ -105,9 +107,213 @@ def test_custom_shard_metadata(tmpdir: Union[str, Path]) -> None: # The object in memory and the saved metadata are the same. assert dataset._dataset_info == Dataset(experiment_path)._dataset_info - # There are three shards with the custom metadata. + # There are four shards: the full custom_metadata_1 shard (closed first), + # followed by the three partial shards closed on exit. shards: list[ShardInfo] = list(dataset.shard_info_iterator(TRAIN_SPLIT)) + assert len(shards) == 4 + assert [(s.custom_metadata, s.number_of_examples) for s in shards] == [ + (custom_metadata_1, 4), + ({}, 1), + (custom_metadata_0, 1), + (custom_metadata_1, 1), + ] + + +def test_custom_metadata_none_and_present_saved_in_separate_shards( + tmpdir: Union[str, Path],) -> None: + """Examples without custom_metadata (`None` or `{}`) and examples with + non-empty custom_metadata must be saved in separate shards regardless of + write order. + """ + values = np.arange(6 * 8, dtype=np.float32).reshape(6, 8) + dataset_path = Path(tmpdir) / "none_vs_present_metadata" + + dataset = Dataset.create( + path=dataset_path, + metadata=Metadata(description="Test None vs non-empty custom_metadata"), + dataset_structure=sedpack.io.metadata.DatasetStructure( + saved_data_description=[ + sedpack.io.metadata.Attribute( + name="x", + dtype="float32", + shape=(8,), + ), + ], + compression="LZ4", + examples_per_shard=4, + shard_file_type="fb", + ), + ) + + meta_a = {"group": "A"} + # Start with non-empty metadata, then None, then {}, then meta_a again. + with dataset.filler(concurrency=2) as filler: + filler.write_example( + values={"x": values[0]}, + split=TRAIN_SPLIT, + custom_metadata=meta_a, + ) + filler.write_example( + values={"x": values[1]}, + split=TRAIN_SPLIT, + custom_metadata=None, + ) + filler.write_example( + values={"x": values[2]}, + split=TRAIN_SPLIT, + custom_metadata={}, + ) + filler.write_example( + values={"x": values[3]}, + split=TRAIN_SPLIT, + custom_metadata=meta_a, + ) + + dataset.check(show_progressbar=False) + shards = list(dataset.shard_info_iterator(TRAIN_SPLIT)) + assert len(shards) == 2 + + shards_by_meta = {bool(s.custom_metadata): s for s in shards} + assert shards_by_meta[False].custom_metadata == {} + assert shards_by_meta[False].number_of_examples == 2 + assert shards_by_meta[True].custom_metadata == meta_a + assert shards_by_meta[True].number_of_examples == 2 + + no_meta_examples = [ + ex["x"] for ex in dataset.as_numpy_iterator( + split=TRAIN_SPLIT, + shuffle=0, + repeat=False, + shard_filter=lambda s: not s.custom_metadata, + ) + ] + meta_a_examples = [ + ex["x"] for ex in dataset.as_numpy_iterator( + split=TRAIN_SPLIT, + shuffle=0, + repeat=False, + shard_filter=lambda s: s.custom_metadata == meta_a, + ) + ] + np.testing.assert_allclose(np.stack(no_meta_examples), values[[1, 2]]) + np.testing.assert_allclose(np.stack(meta_a_examples), values[[0, 3]]) + + +def test_oscillating_custom_metadata_packs_full_shards_within_concurrency( + tmpdir: Union[str, Path],) -> None: + """Oscillating custom_metadata across `<= concurrency` distinct values + reuses open shards and packs full shards instead of creating 1-example + shards, while never exceeding `concurrency` open shards. + """ + # 3 metadata variants * 8 examples each = 24 examples; with + # examples_per_shard=4 and concurrency=3 this should produce exactly + # 6 full shards of 4 examples each (not 24 single-example shards). + values = np.arange(24 * 4, dtype=np.float32).reshape(24, 4) + dataset_path = Path(tmpdir) / "oscillating_metadata_full_shards" + + dataset = Dataset.create( + path=dataset_path, + metadata=Metadata(description="Test oscillating custom_metadata"), + dataset_structure=sedpack.io.metadata.DatasetStructure( + saved_data_description=[ + sedpack.io.metadata.Attribute( + name="x", + dtype="float32", + shape=(4,), + ), + ], + compression="LZ4", + examples_per_shard=4, + shard_file_type="fb", + ), + ) + + metadatas: list[dict[str, Any] | None] = [ + None, + { + "id": 1, + "label": "one" + }, + { + "label": "two", + "id": 2 + }, + ] + + with dataset.filler(concurrency=3) as filler: + for i, val in enumerate(values): + filler.write_example( + values={"x": val}, + split=TRAIN_SPLIT, + custom_metadata=metadatas[i % 3], + ) + assert len(filler._current_shards) <= 3 + + dataset.check(show_progressbar=False) + shards = list(dataset.shard_info_iterator(TRAIN_SPLIT)) + assert len(shards) == 6 + assert all(s.number_of_examples == 4 for s in shards) + + +def test_oscillating_custom_metadata_lru_eviction_and_mutation_safety( + tmpdir: Union[str, Path],) -> None: + """When oscillating across more distinct metadata values than + `concurrency`, LRU eviction bounds `_current_shards` to at most + `concurrency`, and in-place mutation of the caller's dict does not corrupt + previously written shards. + """ + values = np.arange(10 * 4, dtype=np.float32).reshape(10, 4) + dataset_path = Path(tmpdir) / "oscillating_lru_and_mutation" + + dataset = Dataset.create( + path=dataset_path, + metadata=Metadata(description="Test LRU eviction and mutation safety"), + dataset_structure=sedpack.io.metadata.DatasetStructure( + saved_data_description=[ + sedpack.io.metadata.Attribute( + name="x", + dtype="float32", + shape=(4,), + ), + ], + compression="LZ4", + examples_per_shard=4, + shard_file_type="fb", + ), + ) + + mutable_meta: dict[str, Any] = {"nested": {"step": 0}} + with dataset.filler(concurrency=2) as filler: + # Write with step=0, then mutate mutable_meta in place to step=1 and + # step=2. With concurrency=2, step=2 evicts the LRU shard (step=0). + for step in [0, 1, 0, 1, 2]: + mutable_meta["nested"]["step"] = step + filler.write_example( + values={"x": values[step]}, + split=TRAIN_SPLIT, + custom_metadata=mutable_meta, + ) + assert len(filler._current_shards) <= 2 + + dataset.check(show_progressbar=False) + shards = list(dataset.shard_info_iterator(TRAIN_SPLIT)) + # step=0 (2 examples, evicted when step=2 arrived), step=1 (2 examples), + # step=2 (1 example). assert len(shards) == 3 - assert shards[0].custom_metadata == custom_metadata_0 - assert shards[1].custom_metadata == custom_metadata_1 - assert shards[2].custom_metadata == custom_metadata_1 + assert [(s.custom_metadata, s.number_of_examples) for s in shards] == [ + ({ + "nested": { + "step": 0 + } + }, 2), + ({ + "nested": { + "step": 1 + } + }, 2), + ({ + "nested": { + "step": 2 + } + }, 1), + ] diff --git a/tests/io/test_write_multiprocessing.py b/tests/io/test_write_multiprocessing.py index 1b8ce4b4..91df410d 100644 --- a/tests/io/test_write_multiprocessing.py +++ b/tests/io/test_write_multiprocessing.py @@ -17,17 +17,30 @@ import numpy as np import numpy.typing as npt +import pytest import sedpack from sedpack.io import Dataset, DatasetFiller from sedpack.io import Metadata -from sedpack.io.types import SplitT, TRAIN_SPLIT +from sedpack.io.file_info import PathGenerator +from sedpack.io.shard_info_iterator import ( + CachedShardInfoIterator, + ShardInfoIterator, +) +from sedpack.io.types import ( + CompressionT, + HOLDOUT_SPLIT, + ShardFileTypeT, + SplitT, + TEST_SPLIT, + TRAIN_SPLIT, +) +from sedpack.io.utils import is_module_present def feed_writer(dataset_filler: DatasetFiller, array_of_values: npt.NDArray[np.generic], split: SplitT) -> int: # Fill data in the dataset - with dataset_filler as filler: for attribute_value in array_of_values: filler.write_example( @@ -38,12 +51,17 @@ def feed_writer(dataset_filler: DatasetFiller, return len(array_of_values) -def test_write_multiprocessing(tmpdir: Union[str, Path]) -> None: +@pytest.mark.parametrize("single_process", [True, False]) +def test_write_multiprocessing( + tmpdir: Union[str, Path], + single_process: bool, +) -> None: dtype = "float32" array_of_values = np.random.random((1024, 138)) array_of_values = array_of_values.astype(dtype) - tiny_experiment_path: Path = Path(tmpdir) / "write_multiprocessing" + tiny_experiment_path: Path = (Path(tmpdir) / + f"write_multiprocessing_{single_process}") # Create a dataset @@ -92,7 +110,8 @@ def test_write_multiprocessing(tmpdir: Union[str, Path]) -> None: feed_writer=feed_writer, custom_arguments=custom_arguments, custom_kwarguments=custom_kwarguments, - single_process=True) + single_process=single_process, + ) assert results == [ len(part_of_array_of_values[0]) @@ -113,3 +132,422 @@ def test_write_multiprocessing(tmpdir: Union[str, Path]) -> None: # We tested everything assert i + 1 == array_of_values.shape[ 0], "Not all examples have been iterated" + + +@pytest.mark.parametrize( + "shard_file_type,compression", + [ + ("fb", "LZ4"), + ("npz", "ZIP"), + ("tfrec", "GZIP"), + ], +) +def test_filler_multiprocess_all_shard_types( + tmpdir: Union[str, Path], + shard_file_type: ShardFileTypeT, + compression: CompressionT, +) -> None: + if shard_file_type == "tfrec" and not is_module_present("tensorflow"): + pytest.skip("TensorFlow is optional, skip test if not present.") + + array_of_values = np.random.random((60, 16)).astype(np.float32) + dataset_path = Path(tmpdir) / f"filler_mp_{shard_file_type}" + + dataset = Dataset.create( + path=dataset_path, + metadata=Metadata(description="Test multiprocess filler"), + dataset_structure=sedpack.io.metadata.DatasetStructure( + saved_data_description=[ + sedpack.io.metadata.Attribute( + name="attribute_name", + dtype="float32", + shape=(16,), + ), + ], + compression=compression, + examples_per_shard=10, + shard_file_type=shard_file_type, + ), + ) + + with dataset.filler(concurrency=3) as filler: + for attribute_value in array_of_values: + filler.write_example( + values={"attribute_name": attribute_value}, + split=TRAIN_SPLIT, + ) + + reloaded = Dataset(dataset_path) + reloaded.check(show_progressbar=False) + + iterated = [ + ex["attribute_name"] for ex in reloaded.as_numpy_iterator( + split=TRAIN_SPLIT, + shuffle=0, + repeat=False, + ) + ] + assert len(iterated) == len(array_of_values) + np.testing.assert_allclose(np.stack(iterated), array_of_values) + + +def test_filler_multiple_leaf_directories_and_validation( + tmpdir: Union[str, Path],) -> None: + array_of_values = np.arange(20 * 4, dtype=np.float32).reshape(20, 4) + dataset_path = Path(tmpdir) / "filler_multi_leaf" + + dataset = Dataset.create( + path=dataset_path, + metadata=Metadata(description="Test multiple leaf directories"), + dataset_structure=sedpack.io.metadata.DatasetStructure( + saved_data_description=[ + sedpack.io.metadata.Attribute( + name="attribute_name", + dtype="float32", + shape=(4,), + ), + ], + compression="LZ4", + examples_per_shard=4, + shard_file_type="fb", + ), + ) + + with pytest.raises(ValueError, match="directory traversal"): + DatasetFiller( + dataset=dataset, + relative_path_from_split=Path("../escape"), + ) + with pytest.raises(ValueError, match="directory traversal"): + DatasetFiller( + dataset=dataset, + relative_path_from_split=Path("/tmp/absolute_escape"), + ) + + filler = dataset.filler(concurrency=2) + # Force PathGenerator to roll over to a new directory every 2 shards. + filler._path_generator = PathGenerator(max_branching=2, levels=2) + + with filler: + for attribute_value in array_of_values: + filler.write_example( + values={"attribute_name": attribute_value}, + split=TRAIN_SPLIT, + ) + # Exercise the shard_lists property before exit (5 full shards across + # 3 leaf directories: 2, 2, 1 shards per leaf directory). + assert len(filler.shard_lists) == 3 + + # After __exit__, _update_infos merges the 3 leaf directories into a single + # ShardListInfo for `train/shards_list.json`. + updated_infos = filler.get_updated_infos() + assert len(updated_infos) == 1 + assert updated_infos[0].shard_list_info_file.file_path == Path( + "train/shards_list.json") + assert updated_infos[0].number_of_examples == 20 + assert updated_infos[0].number_of_shards == 5 + + reloaded = Dataset(dataset_path) + reloaded.check(show_progressbar=False) + iterated = [ + ex["attribute_name"] for ex in reloaded.as_numpy_iterator( + split=TRAIN_SPLIT, + shuffle=0, + repeat=False, + ) + ] + assert len(iterated) == len(array_of_values) + np.testing.assert_allclose(np.stack(iterated), array_of_values) + + +def test_filler_relative_path_from_split_updates_only_subtree( + tmpdir: Union[str, Path],) -> None: + """When `relative_path_from_split` is non-empty, exiting `DatasetFiller` + updates up to `//shards_list.json` without + modifying `/shards_list.json` or `dataset_info.json` until + `dataset.write_config(updated_infos=...)` is called. + """ + array_of_values = np.arange(20 * 4, dtype=np.float32).reshape(20, 4) + dataset_path = Path(tmpdir) / "filler_subtree_only" + + dataset = Dataset.create( + path=dataset_path, + metadata=Metadata(description="Test subtree-only update"), + dataset_structure=sedpack.io.metadata.DatasetStructure( + saved_data_description=[ + sedpack.io.metadata.Attribute( + name="attribute_name", + dtype="float32", + shape=(4,), + ), + ], + compression="LZ4", + examples_per_shard=4, + shard_file_type="fb", + ), + ) + + worker_0_filler = DatasetFiller( + dataset=Dataset(dataset_path), + concurrency=2, + relative_path_from_split=Path("worker_0"), + ) + # Force multiple leaf directories under train/worker_0/. + worker_0_filler._path_generator = PathGenerator(max_branching=2, levels=2) + + with worker_0_filler: + for attribute_value in array_of_values[:12]: + worker_0_filler.write_example( + values={"attribute_name": attribute_value}, + split=TRAIN_SPLIT, + ) + + worker_0_infos = worker_0_filler.get_updated_infos() + assert len(worker_0_infos) == 1 + assert worker_0_infos[0].shard_list_info_file.file_path == Path( + "train/worker_0/shards_list.json") + assert worker_0_infos[0].number_of_examples == 12 + assert (dataset_path / "train" / "worker_0" / "shards_list.json").is_file() + # Neither `/shards_list.json` nor `dataset_info.json` splits should + # be updated yet. + assert not (dataset_path / "train" / "shards_list.json").exists() + assert not Dataset(dataset_path)._dataset_info.splits + + worker_1_filler = DatasetFiller( + dataset=Dataset(dataset_path), + concurrency=2, + relative_path_from_split=Path("worker_1"), + ) + with worker_1_filler: + for attribute_value in array_of_values[12:]: + worker_1_filler.write_example( + values={"attribute_name": attribute_value}, + split=TRAIN_SPLIT, + ) + + worker_1_infos = worker_1_filler.get_updated_infos() + assert len(worker_1_infos) == 1 + assert worker_1_infos[0].shard_list_info_file.file_path == Path( + "train/worker_1/shards_list.json") + assert worker_1_infos[0].number_of_examples == 8 + + # Coordinator merges worker updates into `/shards_list.json` and + # `dataset_info.json`. + dataset.write_config(updated_infos=worker_0_infos + worker_1_infos) + assert (dataset_path / "train" / "shards_list.json").is_file() + + reloaded = Dataset(dataset_path) + reloaded.check(show_progressbar=False) + iterated = [ + ex["attribute_name"] for ex in reloaded.as_numpy_iterator( + split=TRAIN_SPLIT, + shuffle=0, + repeat=False, + ) + ] + assert len(iterated) == len(array_of_values) + np.testing.assert_allclose(np.stack(iterated), array_of_values) + + +def test_filler_relative_path_multi_split_nested_and_append( + tmpdir: Union[str, Path],) -> None: + """Test nested `relative_path_from_split` across multiple splits and + subsequent appends to an existing worker directory. + """ + train_values = np.arange(16 * 4, dtype=np.float32).reshape(16, 4) + test_values = (np.arange(8 * 4, dtype=np.float32).reshape(8, 4) + 100.0) + val_values = (np.arange(6 * 4, dtype=np.float32).reshape(6, 4) + 200.0) + dataset_path = Path(tmpdir) / "filler_nested_multi_split" + + dataset = Dataset.create( + path=dataset_path, + metadata=Metadata(description="Test nested multi-split workers"), + dataset_structure=sedpack.io.metadata.DatasetStructure( + saved_data_description=[ + sedpack.io.metadata.Attribute( + name="attribute_name", + dtype="float32", + shape=(4,), + ), + ], + compression="LZ4", + examples_per_shard=4, + shard_file_type="fb", + ), + ) + + # Worker 0 writes to both TRAIN_SPLIT and TEST_SPLIT under job_1/worker_0. + w0 = DatasetFiller( + dataset=Dataset(dataset_path), + concurrency=1, + relative_path_from_split=Path("job_1/worker_0"), + ) + with w0: + for val in train_values[:6]: + w0.write_example(values={"attribute_name": val}, split=TRAIN_SPLIT) + for val in test_values: + w0.write_example(values={"attribute_name": val}, split=TEST_SPLIT) + + w0_infos = w0.get_updated_infos() + assert {info.shard_list_info_file.file_path for info in w0_infos} == { + Path("train/job_1/worker_0/shards_list.json"), + Path("test/job_1/worker_0/shards_list.json"), + } + + # Worker 1 writes to TRAIN_SPLIT and HOLDOUT_SPLIT under job_1/worker_1. + w1 = DatasetFiller( + dataset=Dataset(dataset_path), + concurrency=1, + relative_path_from_split=Path("job_1/worker_1"), + ) + with w1: + for val in train_values[6:12]: + w1.write_example(values={"attribute_name": val}, split=TRAIN_SPLIT) + for val in val_values: + w1.write_example(values={"attribute_name": val}, + split=HOLDOUT_SPLIT) + + w1_infos = w1.get_updated_infos() + assert {info.shard_list_info_file.file_path for info in w1_infos} == { + Path("train/job_1/worker_1/shards_list.json"), + Path("holdout/job_1/worker_1/shards_list.json"), + } + + dataset.write_config(updated_infos=w0_infos + w1_infos) + + # Now append more examples into the existing job_1/worker_0 directory. + w0_append = DatasetFiller( + dataset=Dataset(dataset_path), + concurrency=1, + relative_path_from_split=Path("job_1/worker_0"), + ) + with w0_append: + for val in train_values[12:]: + w0_append.write_example( + values={"attribute_name": val}, + split=TRAIN_SPLIT, + ) + + dataset.write_config(updated_infos=w0_append.get_updated_infos()) + + reloaded = Dataset(dataset_path) + reloaded.check(show_progressbar=False) + + iterated_train = [ + ex["attribute_name"] for ex in reloaded.as_numpy_iterator( + split=TRAIN_SPLIT, + shuffle=0, + repeat=False, + ) + ] + iterated_test = [ + ex["attribute_name"] for ex in reloaded.as_numpy_iterator( + split=TEST_SPLIT, + shuffle=0, + repeat=False, + ) + ] + iterated_val = [ + ex["attribute_name"] for ex in reloaded.as_numpy_iterator( + split=HOLDOUT_SPLIT, + shuffle=0, + repeat=False, + ) + ] + + expected_train = np.concatenate( + [train_values[:6], train_values[12:], train_values[6:12]], axis=0) + np.testing.assert_allclose(np.stack(iterated_train), expected_train) + np.testing.assert_allclose(np.stack(iterated_test), test_values) + np.testing.assert_allclose(np.stack(iterated_val), val_values) + + +def test_filler_reentrant_empty_and_sequential_updates( + tmpdir: Union[str, Path],) -> None: + """Test empty `with filler:` exits, re-entering the same `DatasetFiller` + instance multiple times, passing multiple sequential updates for the same + worker directory to `write_config`, and iterating an unwritten split. + """ + values = np.arange(12 * 4, dtype=np.float32).reshape(12, 4) + dataset_path = Path(tmpdir) / "filler_reentrant_and_empty" + + dataset = Dataset.create( + path=dataset_path, + metadata=Metadata(description="Test reentrant and empty filler"), + dataset_structure=sedpack.io.metadata.DatasetStructure( + saved_data_description=[ + sedpack.io.metadata.Attribute( + name="attribute_name", + dtype="float32", + shape=(4,), + ), + ], + compression="LZ4", + examples_per_shard=4, + shard_file_type="fb", + ), + ) + + # Iterating an unwritten split (even with repeat=True) yields 0 shards + # instead of raising KeyError or IndexError. + assert list( + ShardInfoIterator( + dataset_path=dataset.path, + dataset_info=dataset._dataset_info, + split=HOLDOUT_SPLIT, + repeat=False, + )) == [] + assert list( + CachedShardInfoIterator( + dataset_path=dataset.path, + dataset_info=dataset._dataset_info, + split=HOLDOUT_SPLIT, + repeat=True, + )) == [] + + # Exiting a filler without writing any examples is a clean no-op. + empty_filler = DatasetFiller( + dataset=dataset, + relative_path_from_split=Path("empty_worker"), + ) + with empty_filler: + pass + assert empty_filler.get_updated_infos() == [] + assert not (dataset_path / "train" / "empty_worker").exists() + + # Re-entering the same DatasetFiller instance across two `with` blocks and + # passing both `get_updated_infos()` snapshots into a single `write_config` + # call works without assertion errors or duplicate counting. + worker_filler = DatasetFiller( + dataset=Dataset(dataset_path), + concurrency=1, + relative_path_from_split=Path("job_1/worker_0"), + ) + with worker_filler: + for val in values[:6]: + worker_filler.write_example( + values={"attribute_name": val}, + split=TRAIN_SPLIT, + ) + first_infos = worker_filler.get_updated_infos() + + with worker_filler: + for val in values[6:]: + worker_filler.write_example( + values={"attribute_name": val}, + split=TRAIN_SPLIT, + ) + second_infos = worker_filler.get_updated_infos() + + dataset.write_config(updated_infos=first_infos + second_infos) + reloaded = Dataset(dataset_path) + reloaded.check(show_progressbar=False) + iterated = [ + ex["attribute_name"] for ex in reloaded.as_numpy_iterator( + split=TRAIN_SPLIT, + shuffle=0, + repeat=False, + ) + ] + assert len(iterated) == len(values) + np.testing.assert_allclose(np.stack(iterated), values) diff --git a/website/src/content/docs/design/filesystem.mdx b/website/src/content/docs/design/filesystem.mdx index faf51824..fc293e1b 100644 --- a/website/src/content/docs/design/filesystem.mdx +++ b/website/src/content/docs/design/filesystem.mdx @@ -185,9 +185,13 @@ Example intermediate `train/shards_list.json`: ``` When writing examples concurrently or across multiple processes, each writer -writes into its own directory subtree and produces local `shards_list.json` -files. Calling `write_config` runs `merge_shard_infos` to recursively merge -those updates up to `/shards_list.json` and `dataset_info.json`. +writes into its own directory subtree (`//...`) +and merges its local `shards_list.json` files up to +`//shards_list.json` on exit. Calling +`dataset.write_config(updated_infos=...)` on the coordinator then runs +`merge_shard_infos` to merge each worker subtree up to +`/shards_list.json` and `dataset_info.json` (see +[Parallel and Distributed Writing](#parallel-and-distributed-writing) below). ### 4. Individual Shard Metadata (`ShardInfo`) @@ -226,6 +230,98 @@ Example leaf `train/8a745b3853/412a5873e4/20fd9888f6/shards_list.json`: } ``` +## Parallel and Distributed Writing + +`DatasetFiller` decouples in-memory example buffering from shard serialization, +compression, hashing, and metadata merging: + +1. **Single-coordinator writing (`dataset.filler(concurrency=4)`)**: + Each full shard is serialized, compressed, and hashed asynchronously in a + `ProcessPoolExecutor` (`concurrency=4` worker processes by default) across + all shard formats (`"fb"`, `"npz"`, `"tfrec"`). Because + `relative_path_from_split` defaults to `Path(".")` (empty), exiting the + `with` block writes the leaf `shards_list.json` files and automatically + calls `dataset.write_config(...)` to update all the way up to + `/shards_list.json` and `dataset_info.json`. +2. **Built-in multi-process writing (`dataset.write_multiprocessing`)**: + When generating examples in parallel across multiple CPU processes on one + machine, `dataset.write_multiprocessing` assigns each worker a + `DatasetFiller` with a unique non-empty `relative_path_from_split` + (`Path(uuid.uuid4().hex)`). The user-supplied `feed_writer` callable + receives `dataset_filler: DatasetFiller` as its first positional argument, + followed by arbitrary `*args` and `**kwargs` unpacked from that worker's + entry in `custom_arguments` and `custom_kwarguments`. On exit, each worker + updates metadata up to its own `//shards_list.json`, and + `write_multiprocessing` then merges all worker `ShardListInfo` objects into + `/shards_list.json` and `dataset_info.json`: + ```python + def feed_worker( + dataset_filler: DatasetFiller, + chunk: list[ExampleT], + split: SplitT = "train", + ) -> int: + # First argument is `dataset_filler`; remaining `*args` and `**kwargs` + # are unpacked from `custom_arguments` and `custom_kwarguments`. + with dataset_filler: + for values in chunk: + dataset_filler.write_example(values=values, split=split) + return len(chunk) + + counts = dataset.write_multiprocessing( + feed_writer=feed_worker, + custom_arguments=[(chunk_0,), (chunk_1,)], + custom_kwarguments=[{"split": "train"}, {"split": "test"}], + ) + ``` +3. **Custom multi-worker / distributed writing (`DatasetFiller` with non-empty `relative_path_from_split`)**: + When running custom worker processes or distributed jobs, pass a unique, + non-empty `relative_path_from_split` (e.g., `Path(f"worker_{worker_id}")`) + to each worker's `DatasetFiller`. Exiting the `with` block merges and + hashes that worker's shard lists up to + `//shards_list.json` without touching + `/shards_list.json` or `dataset_info.json`. Once all workers finish, + collect `filler.get_updated_infos()` on the coordinator and call + `dataset.write_config(updated_infos=...)`: + ```python + def run_worker( + dataset_path: Path, + worker_id: int, + items: list[ExampleT], + ) -> list[ShardListInfo]: + ds = Dataset(dataset_path) + filler = DatasetFiller( + dataset=ds, + relative_path_from_split=Path(f"worker_{worker_id}"), + ) + with filler: + for item in items: + filler.write_example(values=item, split="train") + return filler.get_updated_infos() + + # Coordinator: merge all worker ShardListInfo updates into the dataset. + updated_infos: list[ShardListInfo] = [] + for worker_id, chunk in enumerate(chunks): + updated_infos.extend(run_worker(dataset.path, worker_id, chunk)) + dataset.write_config(updated_infos=updated_infos) + ``` + + **Caveats for custom multi-worker / distributed writing**: + - `relative_path_from_split` **must be non-empty** for every concurrent + worker even when each worker writes to a different split (e.g., one + worker writing only `"train"` and another writing only `"test"`). If + `relative_path_from_split` is left empty (`Path(".")`), exiting the + `with` block calls `dataset.write_config(...)` and overwrites the shared + root `dataset_info.json`, racing with other workers and losing their + split metadata. + - Concurrent workers must use disjoint `relative_path_from_split` paths + where neither path is a parent prefix of another (e.g., do not run + `Path("a")` and `Path("a/b")` concurrently, since the `Path("a")` worker + writes `/a/shards_list.json` on exit). Sibling paths such as + `Path("job_1/worker_0")` and `Path("job_1/worker_1")` are safe. + - When `relative_path_from_split` is non-empty, the written shards are + not visible in `/shards_list.json` or `dataset_info.json` until + the coordinator calls `dataset.write_config(updated_infos=...)`. + ## Integrity and Merkle Tree Every `FileInfo` stores the hash checksums configured in diff --git a/website/src/content/docs/tutorials/mnist.mdx b/website/src/content/docs/tutorials/mnist.mdx index 5005994c..bfd7b2c3 100644 --- a/website/src/content/docs/tutorials/mnist.mdx +++ b/website/src/content/docs/tutorials/mnist.mdx @@ -203,10 +203,8 @@ information on this see any classical book, e.g., The Elements of Statistical Learning by Jerome H. Friedman, Robert Tibshirani, and Trevor Hastie. An additional detail is that we do not call `write_example` on the dataset -object itself but on a `DatasetFiller` object. This acts as a context manager -which ensures that all data gets saved. A benefit of using a `DatasetFiller` is -that we can create more of those to write in a multithreaded way if we choose -so. +object itself but on a `DatasetFiller` object returned by `dataset.filler()`. +This acts as a context manager which ensures that all data gets saved. The full code is as follows: