Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions project-words.txt
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ Bursztein
CHES
CUDA
Corinna
ENOSPC
Elems
GPAM
Hastie
Expand Down Expand Up @@ -71,6 +72,7 @@ linalg
logpdf
mathbb
mathjax
namedtuple
ndarray
newbyteorder
parseable
Expand All @@ -85,6 +87,7 @@ pyclass
pyfunction
pymethods
pymodule
retryable
riscure
rlib
rngs
Expand All @@ -93,6 +96,7 @@ savez
sedpack
setaf
shardfile
shutil
subclassing
tensorspec
tfdata
Expand Down
3 changes: 3 additions & 0 deletions src/sedpack/io/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,17 @@

from sedpack.io.dataset import Dataset
from sedpack.io.dataset_filler import DatasetFiller
from sedpack.io.errors import DatasetExistsError, DiskFullError
from sedpack.io.metadata import Attribute
from sedpack.io.metadata import DatasetStructure
from sedpack.io.metadata import Metadata

__all__ = [
"Attribute",
"Dataset",
"DatasetExistsError",
"DatasetFiller",
"DatasetStructure",
"DiskFullError",
"Metadata",
]
94 changes: 89 additions & 5 deletions src/sedpack/io/dataset_filler.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,16 @@
from collections import OrderedDict, deque
import concurrent.futures
import copy
import errno
import json
from pathlib import Path
import shutil
from types import TracebackType
from typing import Any, Type, TYPE_CHECKING, get_args

import tenacity

from sedpack.io.errors import DiskFullError
from sedpack.io.file_info import PathGenerator, FileInfo
from sedpack.io.merge_shard_infos import merge_splits_shard_infos
from sedpack.io.shard_file_metadata import ShardInfo, ShardsList, ShardListInfo
Expand All @@ -37,10 +40,15 @@
from sedpack.io.dataset_writing import DatasetWriting


def _is_retryable_exception(exception: BaseException) -> bool:
"""Retry transient OSErrors, but do not retry out-of-disk-space (ENOSPC)."""
return isinstance(exception, OSError) and exception.errno != errno.ENOSPC


@tenacity.retry(
wait=tenacity.wait_random(min=60, max=5 * 60), # 1 to 5 minutes
stop=tenacity.stop_after_attempt(10),
retry=tenacity.retry_if_exception_type(OSError),
retry=tenacity.retry_if_exception(_is_retryable_exception),
reraise=True,
)
def _close_shard(shard: Shard) -> ShardInfo:
Expand Down Expand Up @@ -131,10 +139,12 @@ def run_worker(
"""

def __init__(
self,
dataset: DatasetWriting,
concurrency: int = 4,
relative_path_from_split: Path = Path("."),
self,
dataset: DatasetWriting,
concurrency: int = 4,
relative_path_from_split: Path = Path("."),
*,
max_disk_utilization: float | None = None,
) -> None:
"""Context manager for writing examples into a dataset.

Expand All @@ -157,15 +167,44 @@ def __init__(
`get_updated_infos()` to `dataset.write_config(updated_infos=...)`
(even if each worker writes to a different split). May not contain
"..".

max_disk_utilization (float | None): Maximum fraction of disk capacity
in `(0.0, 1.0]` allowed before raising `DiskFullError`, or `None` to
disable the disk utilization check. Defaults to `None`. Checked at
creation (`__init__`) and in `write_example` before opening a new
shard. When `DiskFullError` is raised during `write_example`,
`__exit__` still flushes and saves all so-far opened shards and
updates the dataset metadata so no previously written data is lost
(which may cause final disk utilization to slightly exceed
`max_disk_utilization`).

Raises:

ValueError: If `relative_path_from_split` contains '..' or
`max_disk_utilization` is outside `(0.0, 1.0]`.

DiskFullError: If disk utilization is already `>=
max_disk_utilization` at creation time.
"""
# Make sure that we do not accidentally traverse the directory above
# `dataset_root_path`.
self._relative_path_from_split: Path = FileInfo.no_directory_traversal(
relative_path_from_split)

if (max_disk_utilization is not None and
not 0.0 < max_disk_utilization <= 1.0):
raise ValueError(
"max_disk_utilization must be in the range (0.0, 1.0] or None, "
f"got {max_disk_utilization}")

self._dataset: DatasetWriting = dataset
self._dataset_root_path: Path = dataset.path
self._dataset_structure: DatasetStructure = dataset.dataset_structure
self._max_disk_utilization: float | None = max_disk_utilization

# Check disk space before spawning the executor pool so we do not leak
# background processes if creation fails.
self._check_disk_space()

self._max_workers: int = max(1, concurrency)
self._concurrent_pool: concurrent.futures.Executor | None = None
Expand All @@ -188,6 +227,41 @@ def __init__(
# Random path generator.
self._path_generator = PathGenerator()

def _check_disk_space(self) -> None:
"""Check that disk utilization is below `self._max_disk_utilization`.

Disk utilization is computed as
`(usage.total - usage.free) / usage.total` so that filesystem blocks
reserved for `root` (e.g., 5% on Linux ext4) are counted as unavailable
to non-root processes (`1.0` corresponds to `usage.free == 0`).

Raises:

DiskFullError: If current disk utilization is `>=
self._max_disk_utilization`.
"""
if self._max_disk_utilization is None:
return

check_path: Path = self._dataset_root_path
while not check_path.exists() and check_path != check_path.parent:
check_path = check_path.parent

usage = shutil.disk_usage(check_path)
if usage.total <= 0:
return
current_utilization: float = (usage.total - usage.free) / usage.total
if current_utilization >= self._max_disk_utilization:
raise DiskFullError(
dataset_path=self._dataset_root_path,
message=(
f"Disk utilization {current_utilization:.1%} at "
f"'{self._dataset_root_path}' reached or exceeded "
f"max_disk_utilization={self._max_disk_utilization:.1%}."),
current_utilization=current_utilization,
max_disk_utilization=self._max_disk_utilization,
)

def _ensure_pool(self) -> concurrent.futures.Executor:
if self._concurrent_pool is None:
self._concurrent_pool = concurrent.futures.ProcessPoolExecutor(
Expand Down Expand Up @@ -259,6 +333,14 @@ def write_example(
custom_metadata (dict[str, Any] | None): Optional JSON-serializable
metadata saved in the shard's `ShardInfo`. Shards can later be
filtered using these metadata.

Raises:

DiskFullError: If a new shard needs to be opened and disk
utilization is at or above `max_disk_utilization`. When used inside
a `DatasetFiller` context manager, all shards opened so far are
still closed and saved to the dataset on context exit (`__exit__`)

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

IIRC now shards live in RAM and are written to disk when closing them. If the disk is already full, current shards won't succeed at being written to the disk unless you take some margin to serialize what's in RAM and not yet on disk, correct?

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes this might be a misleading docstring. Intended use-case is max_disk_utilization=0.8 plus whatever the currently in RAM shards are (should be under 1GB).

The longer version:

The user controls examples_per_shard aiming for decent size shards (10MB to 100MB rule of thumb). The fun part is that the user does not really know how much compression there is (for Magika use-case different filetypes have vastly different compression ratios). The user also controlls concurrency (arg of DatasetFiller.__init__) which tells how many in RAM shards can be open at once -- thus limiting the saving at the end.

I am also not 100% happy with this design but it is enough for automated benchmarking and at the same time not overly complicated.

Alternatives considered:

  • Keeping statistics of closed shards
  • Dropping data (not an option)

before the exception propagates.
"""
if split not in get_args(SplitT):
raise ValueError(f"Not a known split value: {split}")
Expand All @@ -281,6 +363,8 @@ def write_example(
_, evicted_shard = self._current_shards.popitem(last=False)
self.close_shard(shard=evicted_shard)

self._check_disk_space()

current_shard = self._get_new_shard(split=split)
if custom_metadata:
current_shard.shard_info.custom_metadata = copy.deepcopy(
Expand Down
17 changes: 17 additions & 0 deletions src/sedpack/io/dataset_writing.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ def write_multiprocessing( # pylint: disable=too-many-arguments
custom_kwarguments: list[dict[str, Any]] | None = None,
consistency_check: bool = True,
single_process: bool = False,
max_disk_utilization: float | None = None,
) -> list[Any]:
"""Multiprocessing write support. Spawn `len(custom_arguments)`
processes to write examples in parallel. Note that all computation is
Expand Down Expand Up @@ -78,6 +79,10 @@ def write_multiprocessing( # pylint: disable=too-many-arguments
single_process (bool): Debugging run in a single process. Defaults
to False.

max_disk_utilization (float | None): Maximum fraction of disk
capacity in `(0.0, 1.0]` allowed before raising `DiskFullError`, or
`None` to disable the disk utilization check. Defaults to `None`.

Returns: A list of values returned by calls to `feed_writer`.

Example use (see `docs/tutorials/writing/` for more information):
Expand Down Expand Up @@ -131,6 +136,7 @@ def feed_worker(
DatasetFiller(
dataset=self,
relative_path_from_split=Path(uuid.uuid4().hex),
max_disk_utilization=max_disk_utilization,
) for _ in range(len(custom_arguments))
]

Expand Down Expand Up @@ -177,6 +183,7 @@ def feed_worker(
def filler(
self,
concurrency: int = 4,
max_disk_utilization: float | None = None,
) -> DatasetFiller:
"""Return a dataset filler context manager for writing examples.

Expand All @@ -191,6 +198,15 @@ def filler(
concurrency (int): Setting to a positive integer allows writing shard
files in parallel. Defaults to 4.

max_disk_utilization (float | None): Maximum fraction of disk capacity
in `(0.0, 1.0]` allowed before raising `DiskFullError`, or `None` to
disable the disk utilization check. Defaults to `None`. Checked at
creation (`__init__`) and in `write_example` before opening a new
shard. When `DiskFullError` is raised during `write_example`,
`DatasetFiller.__exit__` still flushes and saves all so-far opened
shards so no previously written data is lost (which may cause final
disk utilization to slightly exceed `max_disk_utilization`).

Example use:
# Context manager properly opens and closes shards.
with dataset.filler() as dataset_filler:
Expand All @@ -204,6 +220,7 @@ def filler(
return DatasetFiller(
dataset=self,
concurrency=concurrency,
max_disk_utilization=max_disk_utilization,
)

def write_config(
Expand Down
37 changes: 30 additions & 7 deletions src/sedpack/io/errors.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2023 Google LLC
# Copyright 2023-2025 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
Expand All @@ -11,13 +11,9 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Implement an error to indicate that a scaaml.io.Dataset already exists.

Creating scaaml.io.Dataset should not overwrite existing files. When it could
the constructor needs to raise an error, which should also contain the dataset
directory.
"""
"""Custom exceptions for sedpack.io."""

import errno
from pathlib import Path


Expand All @@ -34,3 +30,30 @@ def __init__(self, dataset_path: Path) -> None:
f'Dataset info file exists and would be overwritten. Use instead:'
f' Dataset(path="{dataset_path}")')
self.dataset_path = dataset_path


class DiskFullError(OSError):
"""Error for signalling that disk utilization is at or above the safety
threshold.
"""

def __init__(
self,
dataset_path: Path,
message: str,
*,
current_utilization: float | None = None,
max_disk_utilization: float | None = None,
) -> None:
"""Represents that the disk does not have enough free space.

Args:
dataset_path: The dataset path.
message: Human-readable error message.
current_utilization: Current disk utilization ratio in [0.0, 1.0].
max_disk_utilization: Maximum allowed disk utilization ratio.
"""
super().__init__(errno.ENOSPC, message)
self.dataset_path = dataset_path
self.current_utilization = current_utilization
self.max_disk_utilization = max_disk_utilization
4 changes: 3 additions & 1 deletion src/sedpack/io/shard/shard_writer_flatbuffer.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
For information about FlatBuffers see https://flatbuffers.dev/
"""

import errno
from pathlib import Path

from flatbuffers import Builder
Expand Down Expand Up @@ -189,7 +190,8 @@ def save_numpy_vector_as_bytearray( # type: ignore[no-any-unimported]
@tenacity.retry(
wait=tenacity.wait_random(min=60, max=5 * 60), # 1 to 5 minutes
stop=tenacity.stop_after_attempt(10),
retry=tenacity.retry_if_exception_type(OSError),
retry=tenacity.retry_if_exception(
lambda exc: isinstance(exc, OSError) and exc.errno != errno.ENOSPC),
reraise=True,
)
def _write_file(self, file_content: bytes) -> None:
Expand Down
Loading
Loading