diff --git a/project-words.txt b/project-words.txt index 20994953..9b10ecbf 100644 --- a/project-words.txt +++ b/project-words.txt @@ -5,6 +5,7 @@ Bursztein CHES CUDA Corinna +ENOSPC Elems GPAM Hastie @@ -71,6 +72,7 @@ linalg logpdf mathbb mathjax +namedtuple ndarray newbyteorder parseable @@ -85,6 +87,7 @@ pyclass pyfunction pymethods pymodule +retryable riscure rlib rngs @@ -93,6 +96,7 @@ savez sedpack setaf shardfile +shutil subclassing tensorspec tfdata diff --git a/src/sedpack/io/__init__.py b/src/sedpack/io/__init__.py index a6b35937..6802f2d3 100644 --- a/src/sedpack/io/__init__.py +++ b/src/sedpack/io/__init__.py @@ -15,6 +15,7 @@ from sedpack.io.dataset import Dataset from sedpack.io.dataset_filler import DatasetFiller +from sedpack.io.errors import DatasetExistsError, DiskFullError from sedpack.io.metadata import Attribute from sedpack.io.metadata import DatasetStructure from sedpack.io.metadata import Metadata @@ -22,7 +23,9 @@ __all__ = [ "Attribute", "Dataset", + "DatasetExistsError", "DatasetFiller", "DatasetStructure", + "DiskFullError", "Metadata", ] diff --git a/src/sedpack/io/dataset_filler.py b/src/sedpack/io/dataset_filler.py index 2700163f..83f2ce06 100644 --- a/src/sedpack/io/dataset_filler.py +++ b/src/sedpack/io/dataset_filler.py @@ -18,13 +18,16 @@ from collections import OrderedDict, deque import concurrent.futures import copy +import errno import json from pathlib import Path +import shutil from types import TracebackType from typing import Any, Type, TYPE_CHECKING, get_args import tenacity +from sedpack.io.errors import DiskFullError from sedpack.io.file_info import PathGenerator, FileInfo from sedpack.io.merge_shard_infos import merge_splits_shard_infos from sedpack.io.shard_file_metadata import ShardInfo, ShardsList, ShardListInfo @@ -37,10 +40,15 @@ from sedpack.io.dataset_writing import DatasetWriting +def _is_retryable_exception(exception: BaseException) -> bool: + """Retry transient OSErrors, but do not retry out-of-disk-space (ENOSPC).""" + return isinstance(exception, OSError) and exception.errno != errno.ENOSPC + + @tenacity.retry( wait=tenacity.wait_random(min=60, max=5 * 60), # 1 to 5 minutes stop=tenacity.stop_after_attempt(10), - retry=tenacity.retry_if_exception_type(OSError), + retry=tenacity.retry_if_exception(_is_retryable_exception), reraise=True, ) def _close_shard(shard: Shard) -> ShardInfo: @@ -131,10 +139,12 @@ def run_worker( """ def __init__( - self, - dataset: DatasetWriting, - concurrency: int = 4, - relative_path_from_split: Path = Path("."), + self, + dataset: DatasetWriting, + concurrency: int = 4, + relative_path_from_split: Path = Path("."), + *, + max_disk_utilization: float | None = None, ) -> None: """Context manager for writing examples into a dataset. @@ -157,15 +167,44 @@ def __init__( `get_updated_infos()` to `dataset.write_config(updated_infos=...)` (even if each worker writes to a different split). May not contain "..". + + max_disk_utilization (float | None): Maximum fraction of disk capacity + in `(0.0, 1.0]` allowed before raising `DiskFullError`, or `None` to + disable the disk utilization check. Defaults to `None`. Checked at + creation (`__init__`) and in `write_example` before opening a new + shard. When `DiskFullError` is raised during `write_example`, + `__exit__` still flushes and saves all so-far opened shards and + updates the dataset metadata so no previously written data is lost + (which may cause final disk utilization to slightly exceed + `max_disk_utilization`). + + Raises: + + ValueError: If `relative_path_from_split` contains '..' or + `max_disk_utilization` is outside `(0.0, 1.0]`. + + DiskFullError: If disk utilization is already `>= + max_disk_utilization` at creation time. """ # Make sure that we do not accidentally traverse the directory above # `dataset_root_path`. self._relative_path_from_split: Path = FileInfo.no_directory_traversal( relative_path_from_split) + if (max_disk_utilization is not None and + not 0.0 < max_disk_utilization <= 1.0): + raise ValueError( + "max_disk_utilization must be in the range (0.0, 1.0] or None, " + f"got {max_disk_utilization}") + self._dataset: DatasetWriting = dataset self._dataset_root_path: Path = dataset.path self._dataset_structure: DatasetStructure = dataset.dataset_structure + self._max_disk_utilization: float | None = max_disk_utilization + + # Check disk space before spawning the executor pool so we do not leak + # background processes if creation fails. + self._check_disk_space() self._max_workers: int = max(1, concurrency) self._concurrent_pool: concurrent.futures.Executor | None = None @@ -188,6 +227,41 @@ def __init__( # Random path generator. self._path_generator = PathGenerator() + def _check_disk_space(self) -> None: + """Check that disk utilization is below `self._max_disk_utilization`. + + Disk utilization is computed as + `(usage.total - usage.free) / usage.total` so that filesystem blocks + reserved for `root` (e.g., 5% on Linux ext4) are counted as unavailable + to non-root processes (`1.0` corresponds to `usage.free == 0`). + + Raises: + + DiskFullError: If current disk utilization is `>= + self._max_disk_utilization`. + """ + if self._max_disk_utilization is None: + return + + check_path: Path = self._dataset_root_path + while not check_path.exists() and check_path != check_path.parent: + check_path = check_path.parent + + usage = shutil.disk_usage(check_path) + if usage.total <= 0: + return + current_utilization: float = (usage.total - usage.free) / usage.total + if current_utilization >= self._max_disk_utilization: + raise DiskFullError( + dataset_path=self._dataset_root_path, + message=( + f"Disk utilization {current_utilization:.1%} at " + f"'{self._dataset_root_path}' reached or exceeded " + f"max_disk_utilization={self._max_disk_utilization:.1%}."), + current_utilization=current_utilization, + max_disk_utilization=self._max_disk_utilization, + ) + def _ensure_pool(self) -> concurrent.futures.Executor: if self._concurrent_pool is None: self._concurrent_pool = concurrent.futures.ProcessPoolExecutor( @@ -259,6 +333,14 @@ def write_example( custom_metadata (dict[str, Any] | None): Optional JSON-serializable metadata saved in the shard's `ShardInfo`. Shards can later be filtered using these metadata. + + Raises: + + DiskFullError: If a new shard needs to be opened and disk + utilization is at or above `max_disk_utilization`. When used inside + a `DatasetFiller` context manager, all shards opened so far are + still closed and saved to the dataset on context exit (`__exit__`) + before the exception propagates. """ if split not in get_args(SplitT): raise ValueError(f"Not a known split value: {split}") @@ -281,6 +363,8 @@ def write_example( _, evicted_shard = self._current_shards.popitem(last=False) self.close_shard(shard=evicted_shard) + self._check_disk_space() + current_shard = self._get_new_shard(split=split) if custom_metadata: current_shard.shard_info.custom_metadata = copy.deepcopy( diff --git a/src/sedpack/io/dataset_writing.py b/src/sedpack/io/dataset_writing.py index 97fbbafa..64a961ef 100644 --- a/src/sedpack/io/dataset_writing.py +++ b/src/sedpack/io/dataset_writing.py @@ -47,6 +47,7 @@ def write_multiprocessing( # pylint: disable=too-many-arguments custom_kwarguments: list[dict[str, Any]] | None = None, consistency_check: bool = True, single_process: bool = False, + max_disk_utilization: float | None = None, ) -> list[Any]: """Multiprocessing write support. Spawn `len(custom_arguments)` processes to write examples in parallel. Note that all computation is @@ -78,6 +79,10 @@ def write_multiprocessing( # pylint: disable=too-many-arguments single_process (bool): Debugging run in a single process. Defaults to False. + max_disk_utilization (float | None): Maximum fraction of disk + capacity in `(0.0, 1.0]` allowed before raising `DiskFullError`, or + `None` to disable the disk utilization check. Defaults to `None`. + Returns: A list of values returned by calls to `feed_writer`. Example use (see `docs/tutorials/writing/` for more information): @@ -131,6 +136,7 @@ def feed_worker( DatasetFiller( dataset=self, relative_path_from_split=Path(uuid.uuid4().hex), + max_disk_utilization=max_disk_utilization, ) for _ in range(len(custom_arguments)) ] @@ -177,6 +183,7 @@ def feed_worker( def filler( self, concurrency: int = 4, + max_disk_utilization: float | None = None, ) -> DatasetFiller: """Return a dataset filler context manager for writing examples. @@ -191,6 +198,15 @@ def filler( concurrency (int): Setting to a positive integer allows writing shard files in parallel. Defaults to 4. + max_disk_utilization (float | None): Maximum fraction of disk capacity + in `(0.0, 1.0]` allowed before raising `DiskFullError`, or `None` to + disable the disk utilization check. Defaults to `None`. Checked at + creation (`__init__`) and in `write_example` before opening a new + shard. When `DiskFullError` is raised during `write_example`, + `DatasetFiller.__exit__` still flushes and saves all so-far opened + shards so no previously written data is lost (which may cause final + disk utilization to slightly exceed `max_disk_utilization`). + Example use: # Context manager properly opens and closes shards. with dataset.filler() as dataset_filler: @@ -204,6 +220,7 @@ def filler( return DatasetFiller( dataset=self, concurrency=concurrency, + max_disk_utilization=max_disk_utilization, ) def write_config( diff --git a/src/sedpack/io/errors.py b/src/sedpack/io/errors.py index f0ba8c8f..db0e5cc5 100644 --- a/src/sedpack/io/errors.py +++ b/src/sedpack/io/errors.py @@ -1,4 +1,4 @@ -# Copyright 2023 Google LLC +# Copyright 2023-2025 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,13 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Implement an error to indicate that a scaaml.io.Dataset already exists. - -Creating scaaml.io.Dataset should not overwrite existing files. When it could -the constructor needs to raise an error, which should also contain the dataset -directory. -""" +"""Custom exceptions for sedpack.io.""" +import errno from pathlib import Path @@ -34,3 +30,30 @@ def __init__(self, dataset_path: Path) -> None: f'Dataset info file exists and would be overwritten. Use instead:' f' Dataset(path="{dataset_path}")') self.dataset_path = dataset_path + + +class DiskFullError(OSError): + """Error for signalling that disk utilization is at or above the safety + threshold. + """ + + def __init__( + self, + dataset_path: Path, + message: str, + *, + current_utilization: float | None = None, + max_disk_utilization: float | None = None, + ) -> None: + """Represents that the disk does not have enough free space. + + Args: + dataset_path: The dataset path. + message: Human-readable error message. + current_utilization: Current disk utilization ratio in [0.0, 1.0]. + max_disk_utilization: Maximum allowed disk utilization ratio. + """ + super().__init__(errno.ENOSPC, message) + self.dataset_path = dataset_path + self.current_utilization = current_utilization + self.max_disk_utilization = max_disk_utilization diff --git a/src/sedpack/io/shard/shard_writer_flatbuffer.py b/src/sedpack/io/shard/shard_writer_flatbuffer.py index 7ec1b2f9..88693683 100644 --- a/src/sedpack/io/shard/shard_writer_flatbuffer.py +++ b/src/sedpack/io/shard/shard_writer_flatbuffer.py @@ -16,6 +16,7 @@ For information about FlatBuffers see https://flatbuffers.dev/ """ +import errno from pathlib import Path from flatbuffers import Builder @@ -189,7 +190,8 @@ def save_numpy_vector_as_bytearray( # type: ignore[no-any-unimported] @tenacity.retry( wait=tenacity.wait_random(min=60, max=5 * 60), # 1 to 5 minutes stop=tenacity.stop_after_attempt(10), - retry=tenacity.retry_if_exception_type(OSError), + retry=tenacity.retry_if_exception( + lambda exc: isinstance(exc, OSError) and exc.errno != errno.ENOSPC), reraise=True, ) def _write_file(self, file_content: bytes) -> None: diff --git a/tests/io/test_error.py b/tests/io/test_error.py index e52704e7..f5fb9022 100644 --- a/tests/io/test_error.py +++ b/tests/io/test_error.py @@ -1,4 +1,4 @@ -# Copyright 2024 Google LLC +# Copyright 2024-2025 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,19 +11,26 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +"""Tests for custom dataset errors and disk-space checks.""" +from collections import namedtuple from pathlib import Path +from unittest.mock import patch + +import numpy as np import pytest import sedpack -from sedpack.io import Dataset -from sedpack.io import Metadata -from sedpack.io.errors import DatasetExistsError +from sedpack.io import Dataset, DatasetFiller, Metadata +from sedpack.io.errors import DatasetExistsError, DiskFullError +_DiskUsage = namedtuple("_DiskUsage", ["total", "used", "free"]) -def test_dataset_exists(tmpdir: str | Path) -> None: - tiny_experiment_path: Path = Path(tmpdir) / "exists" - # Metadata + +def _create_test_dataset( + dataset_path: Path, + examples_per_shard: int = 4, +) -> Dataset: dataset_metadata = Metadata(description="Test of the lib") example_attributes = [ sedpack.io.metadata.Attribute( @@ -33,19 +40,265 @@ def test_dataset_exists(tmpdir: str | Path) -> None: ), ] dataset_structure = sedpack.io.metadata.DatasetStructure( - saved_data_description=example_attributes) - - # First should be ok. - dataset = Dataset.create( - path=tiny_experiment_path, + saved_data_description=example_attributes, + examples_per_shard=examples_per_shard, + ) + return Dataset.create( + path=dataset_path, metadata=dataset_metadata, dataset_structure=dataset_structure, ) + +def test_dataset_exists(tmpdir: str | Path) -> None: + tiny_experiment_path: Path = Path(tmpdir) / "exists" + _create_test_dataset(tiny_experiment_path) + with pytest.raises(DatasetExistsError): # This would overwrite. - dataset = Dataset.create( - path=tiny_experiment_path, - metadata=dataset_metadata, - dataset_structure=dataset_structure, + _create_test_dataset(tiny_experiment_path) + + +@pytest.mark.parametrize("invalid_utilization", [0.0, -0.1, 1.01, 2.0]) +def test_filler_invalid_max_disk_utilization( + tmpdir: str | Path, + invalid_utilization: float, +) -> None: + dataset = _create_test_dataset(Path(tmpdir) / "invalid_utilization") + with pytest.raises(ValueError, match="max_disk_utilization"): + dataset.filler(max_disk_utilization=invalid_utilization) + + +def test_filler_walks_up_to_existing_parent_and_zero_total( + tmpdir: str | Path,) -> None: + base_dir = Path(tmpdir) + dataset = _create_test_dataset(base_dir / "existing_dataset") + dataset.path = base_dir / "nested" / "not" / "created" / "yet" + + checked_paths: list[Path] = [] + + def fake_disk_usage(path: Path) -> _DiskUsage: + checked_paths.append(path) + return _DiskUsage(total=0, used=0, free=0) + + with patch( + "sedpack.io.dataset_filler.shutil.disk_usage", + side_effect=fake_disk_usage, + ): + filler = DatasetFiller( + dataset=dataset, + max_disk_utilization=0.8, ) + assert checked_paths == [base_dir] + del filler + + +def test_filler_raises_at_creation_when_disk_full(tmpdir: str | Path) -> None: + dataset = _create_test_dataset(Path(tmpdir) / "disk_full_at_creation") + + # 80% unavailable (including root-reserved blocks: used=750, free=200) -> + # utilization is (1000 - 200) / 1000 = 0.8 >= 0.8. + with patch( + "sedpack.io.dataset_filler.shutil.disk_usage", + return_value=_DiskUsage(total=1000, used=750, free=200), + ): + with pytest.raises(DiskFullError) as exc_info: + dataset.filler(max_disk_utilization=0.8) + assert exc_info.value.current_utilization == pytest.approx(0.8) + assert exc_info.value.max_disk_utilization == pytest.approx(0.8) + + # Default (max_disk_utilization=None) disables check even when free == 0. + with patch( + "sedpack.io.dataset_filler.shutil.disk_usage", + return_value=_DiskUsage(total=1000, used=1000, free=0), + ): + with dataset.filler() as filler: + filler.write_example( + values={"attribute_name": np.ones(10, dtype=np.float32)}, + split="train", + ) + + +def test_filler_saves_opened_shards_when_disk_fills_during_write( + tmpdir: str | Path,) -> None: + """When write_example raises DiskFullError before opening a new shard, + DatasetFiller.__exit__ must still flush and save all so-far opened shards + (both previously closed full shards and partial open shards across multiple + splits) without losing data. + """ + dataset = _create_test_dataset( + Path(tmpdir) / "disk_full_mid_write", + examples_per_shard=4, + ) + + train_data = np.arange(90, dtype=np.float32).reshape(9, 10) + test_data = (np.arange(30, dtype=np.float32) + 1000.0).reshape(3, 10) + + current_usage = _DiskUsage(total=1000, used=500, free=500) + + def fake_disk_usage(unused_path: Path) -> _DiskUsage: + _ = unused_path + return current_usage + + with patch( + "sedpack.io.dataset_filler.shutil.disk_usage", + side_effect=fake_disk_usage, + ): + with pytest.raises(DiskFullError): + with dataset.filler(max_disk_utilization=0.8) as filler: + # Write 8 train examples (2 full shards of 4). + for i in range(8): + filler.write_example( + values={"attribute_name": train_data[i]}, + split="train", + ) + # Write 3 test examples (1 partial open shard of 3). + for i in range(3): + filler.write_example( + values={"attribute_name": test_data[i]}, + split="test", + ) + + # Simulate disk filling up to 85% before the 9th train example + # (which attempts to open a 3rd train shard after the 2nd train + # shard was closed on the 8th example). + current_usage = _DiskUsage(total=1000, used=850, free=150) + filler.write_example( + values={"attribute_name": train_data[8]}, + split="train", + ) + + # Attempting to create a new filler while disk is still at 85% must + # raise immediately at creation time. + with pytest.raises(DiskFullError): + dataset.filler(max_disk_utilization=0.8) + + # Verify dataset integrity and that all 8 train + 3 test examples were + # saved without data loss. + dataset.check(show_progressbar=False) + reloaded = Dataset(dataset.path) + reloaded.check(show_progressbar=False) + + saved_train = [ + ex["attribute_name"] for ex in reloaded.as_numpy_iterator( + split="train", + shuffle=0, + repeat=False, + ) + ] + assert len(saved_train) == 8 + np.testing.assert_allclose(np.stack(saved_train), train_data[:8]) + + saved_test = [ + ex["attribute_name"] for ex in reloaded.as_numpy_iterator( + split="test", + shuffle=0, + repeat=False, + ) + ] + assert len(saved_test) == 3 + np.testing.assert_allclose(np.stack(saved_test), test_data) + + +def test_filler_disk_full_right_after_shard_rotation_does_not_double_close( + tmpdir: str | Path,) -> None: + """If disk utilization crosses max_disk_utilization after closing a full + shard and before opening the next shard during write_example, __exit__ + must not close that shard a second time and the closed shard must be saved + properly. + """ + dataset = _create_test_dataset( + Path(tmpdir) / "disk_full_after_rotation", + examples_per_shard=2, + ) + train_data = np.arange(30, dtype=np.float32).reshape(3, 10) + + # Calls to disk_usage: + # 1. DatasetFiller.__init__ -> ok + # 2. write_example #1 (before opening shard #1) -> ok + # 3. write_example #3 (after shard #1 closed on #2, before opening shard #2) + # -> disk full (85%)! + usages = iter([ + _DiskUsage(total=1000, used=500, free=500), + _DiskUsage(total=1000, used=500, free=500), + _DiskUsage(total=1000, used=850, free=150), + ]) + + with patch( + "sedpack.io.dataset_filler.shutil.disk_usage", + side_effect=lambda unused_path: next(usages), + ): + with pytest.raises(DiskFullError): + with dataset.filler(max_disk_utilization=0.8) as filler: + for i in range(3): + filler.write_example( + values={"attribute_name": train_data[i]}, + split="train", + ) + + reloaded = Dataset(dataset.path) + reloaded.check(show_progressbar=False) + saved_train = [ + ex["attribute_name"] for ex in reloaded.as_numpy_iterator( + split="train", + shuffle=0, + repeat=False, + ) + ] + assert len(saved_train) == 2 + np.testing.assert_allclose(np.stack(saved_train), train_data[:2]) + + +def test_filler_disk_full_on_lru_eviction_saves_evicted_and_open_shards( + tmpdir: str | Path,) -> None: + """If LRU shard eviction closes a shard and DiskFullError is then raised + before opening the new shard, both the evicted shard and any remaining open + shards in the LRU cache are flushed and saved properly on __exit__. + """ + dataset = _create_test_dataset( + Path(tmpdir) / "disk_full_lru_eviction", + examples_per_shard=4, + ) + train_data = np.arange(30, dtype=np.float32).reshape(3, 10) + + # concurrency=2 allows 2 open shards in _current_shards. + # Calls to disk_usage: + # 1. DatasetFiller.__init__ -> ok + # 2. write_example #1 (custom_metadata={"group": 0}) -> ok + # 3. write_example #2 (custom_metadata={"group": 1}) -> ok + # 4. write_example #3 (custom_metadata={"group": 2}, evicts group 0 and + # checks disk space before opening group 2) -> disk full (85%)! + usages = iter([ + _DiskUsage(total=1000, used=500, free=500), + _DiskUsage(total=1000, used=500, free=500), + _DiskUsage(total=1000, used=500, free=500), + _DiskUsage(total=1000, used=850, free=150), + ]) + + with patch( + "sedpack.io.dataset_filler.shutil.disk_usage", + side_effect=lambda unused_path: next(usages), + ): + with pytest.raises(DiskFullError): + with dataset.filler( + concurrency=2, + max_disk_utilization=0.8, + ) as filler: + for i in range(3): + filler.write_example( + values={"attribute_name": train_data[i]}, + split="train", + custom_metadata={"group": i}, + ) + + reloaded = Dataset(dataset.path) + reloaded.check(show_progressbar=False) + saved_train = [ + ex["attribute_name"] for ex in reloaded.as_numpy_iterator( + split="train", + shuffle=0, + repeat=False, + ) + ] + assert len(saved_train) == 2 + np.testing.assert_allclose(np.stack(saved_train), train_data[:2])