Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 35 additions & 23 deletions src/sedpack/io/shard/shard_writer_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,9 @@


class ShardWriterBase(ABC):
"""Shard writing capabilities.
"""Base class for shard writing. Buffers examples in memory so the shard
is pickleable and format-specific serialization in `close()` can run in a
different process.
"""

def __init__(self, dataset_structure: DatasetStructure,
Expand All @@ -42,9 +44,7 @@ def __init__(self, dataset_structure: DatasetStructure,
# Information needed to save the shard.
self.dataset_structure: DatasetStructure = dataset_structure
self._shard_file: Path = shard_file

# Make sure that the directory exists.
self._shard_file.parent.mkdir(exist_ok=True, parents=True)
self._examples: list[ExampleT] = []

# Make sure that the compression is supported for this shard file type.
assert dataset_structure.compression in self.supported_compressions()
Expand Down Expand Up @@ -96,17 +96,18 @@ def attribute_value_to_bytes(
return np.asarray(value_np, dtype=np.uint8, order="C").tobytes()

def write(self, values: ExampleT) -> None:
"""Write an example on disk. Writing may be buffered.
"""Buffer an example in memory to be serialized on `close()`.

Args:

values (ExampleT): Attribute values.
"""
example: ExampleT = {}
# Check the values are correct type and shape.
for attribute in self.dataset_structure.saved_data_description:
value = values[attribute.name]
if attribute.dtype == "bytes":
self.attribute_value_to_bytes(
example[attribute.name] = self.attribute_value_to_bytes(
attribute=attribute,
value=value,
)
Expand All @@ -119,30 +120,41 @@ def write(self, values: ExampleT) -> None:

# If the attribute dtype is "bytes" and shape is empty tuple we
# consider this a variable size attribute and do not check shape.
if attribute.has_variable_size():
continue
if not attribute.has_variable_size():
# Else check the shape (the value should be a NumPy array but
# maybe it is an int or bytearray).
current_shape = np.asarray(value).shape
if current_shape != attribute.shape:
raise ValueError(
f"Attribute {attribute.name} has shape "
f"{current_shape} expected {attribute.shape}")

# Else check the shape (the value should be a NumPy array but maybe
# it is an int or bytearray).
current_shape = np.asarray(value).shape
if current_shape != attribute.shape:
raise ValueError(f"Attribute {attribute.name} has shape "
f"{current_shape} expected {attribute.shape}")
example[attribute.name] = (np.copy(value) if isinstance(
value, np.ndarray) else value)

self._write(values=values)
self._examples.append(example)

@abstractmethod
def _write(self, values: ExampleT) -> None:
"""Write an example on disk. Writing may be buffered.
def close(self) -> tuple[str, ...]:
"""Serialize buffered examples into the shard file(-s) and return hash
checksums.
"""
if not self._examples:
# Nothing to save.
assert not self._shard_file.is_file()
return ()

Args:
# Make sure that the directory exists before writing the shard file.
self._shard_file.parent.mkdir(exist_ok=True, parents=True)

values (ExampleT): Attribute values.
"""
hash_checksums = self._close()
self._examples.clear()
assert self._shard_file.is_file()
return hash_checksums

@abstractmethod
def close(self) -> tuple[str, ...]:
"""Close the shard file(-s).
def _close(self) -> tuple[str, ...]:
"""Implementation of shard file(-s) serialization for non-empty
`self._examples`.
"""

@staticmethod
Expand Down
133 changes: 64 additions & 69 deletions src/sedpack/io/shard/shard_writer_flatbuffer.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@

from sedpack.io.compress import CompressedFile
from sedpack.io.metadata import Attribute, DatasetStructure
from sedpack.io.types import AttributeValueT, CompressionT, ExampleT
from sedpack.io.types import AttributeValueT, CompressionT
from sedpack.io.shard.shard_writer_base import ShardWriterBase
from sedpack.io.utils import hash_checksums_from_bytes

Expand Down Expand Up @@ -56,51 +56,6 @@ def __init__(self, dataset_structure: DatasetStructure,
shard_file=shard_file,
)

# Offsets.
self._examples: list[int] = []

self._builder: Builder | None = None # type: ignore[no-any-unimported]

def _write(self, values: ExampleT) -> None:
"""Write an example on disk. Writing may be buffered.

Args:

values (ExampleT): Attribute values.
"""
if self._builder is None:
self._builder = Builder(0)

# Since we are not saving attribute names we need to make sure to
# iterate in the correct order.
saved_attributes: list[int] = []
for attribute in self.dataset_structure.saved_data_description:
attribute_bytes: int = self.save_numpy_vector_as_bytearray(
builder=self._builder,
attribute=attribute,
value=values[attribute.name],
)

fbapi_Attribute.AttributeStart(self._builder)
fbapi_Attribute.AttributeAddAttributeBytes(
self._builder,
attribute_bytes,
)
saved_attributes.append(fbapi_Attribute.AttributeEnd(self._builder))

# Save attributes vector.
fbapi_Example.ExampleStartAttributesVector(self._builder,
len(saved_attributes))
for offset in reversed(saved_attributes):
self._builder.PrependUOffsetTRelative(offset)
attributes_vector_offset = self._builder.EndVector()

# Save the example.
fbapi_Example.ExampleStart(self._builder)
fbapi_Example.ExampleAddAttributes(self._builder,
attributes_vector_offset)
self._examples.append(fbapi_Example.ExampleEnd(self._builder))

@staticmethod
def _np_to_bytes(
attribute: Attribute,
Expand Down Expand Up @@ -205,6 +160,10 @@ def save_numpy_vector_as_bytearray( # type: ignore[no-any-unimported]
length: int = len(byte_representation)

# Start a vector and move head accordingly.
# FlatBuffers grows `builder.Bytes` backwards from the end toward index
# 0. `StartVector` ensures capacity and aligns the write cursor
# (`builder.Head()`, an index from the start of `builder.Bytes`), so
# decrementing `head` by `length` reserves the slice for a bulk copy.
builder.StartVector(
elemSize=1, # Storing bytes.
numElems=length,
Expand All @@ -221,51 +180,87 @@ def save_numpy_vector_as_bytearray( # type: ignore[no-any-unimported]
builder.vectorNumElems = length

# Return the vector offset.
# `EndVector` prepends the 4-byte element count (`vectorNumElems`) and
# returns a `UOffsetT` offset measured from the *end* of `builder.Bytes`
# (`len(builder.Bytes) - builder.Head()`), which remains valid if the
# buffer is later reallocated at the front.
return builder.EndVector() # type: ignore[no-any-return]

@tenacity.retry(
wait=tenacity.wait_random(min=60, max=5 * 60), # 1 to 5 minutes
stop=tenacity.stop_after_attempt(10),
retry=tenacity.retry_if_exception_type(OSError),
reraise=True,
)
def _write_file(self, file_content: bytes) -> None:
# Write the buffer into a file.
with open(self._shard_file, "wb") as file:
file.write(file_content)

def close(self) -> tuple[str, ...]:
"""Close the shard file(-s).
def _close(self) -> tuple[str, ...]:
"""Serialize buffered examples into a FlatBuffer shard file.
"""
if not self._examples:
# Nothing to save.
assert not self._shard_file.is_file()
return hash_checksums_from_bytes(
file_content=b"",
hashes=self.dataset_structure.hash_checksum_algorithms,
)
builder = Builder(0)

if not self._builder:
raise ValueError("Attempting to close a closed shard")
# Since we are not saving attribute names we need to make sure to
# iterate in the correct order.
# FlatBuffers forbids nested table/vector construction (`Start...` /
# `End...` cannot be interleaved). Child objects must be built bottom-up
# first (byte vector -> Attribute table -> Attributes vector -> Example
# table -> Examples vector -> Shard table) and referenced via their
# buffer-end-relative `UOffsetT` offsets.
examples: list[int] = []
for example in self._examples:
saved_attributes: list[int] = []
for attribute in self.dataset_structure.saved_data_description:
attribute_bytes: int = self.save_numpy_vector_as_bytearray(
builder=builder,
attribute=attribute,
value=example[attribute.name],
)
fbapi_Attribute.AttributeStart(builder)
fbapi_Attribute.AttributeAddAttributeBytes(
builder,
attribute_bytes,
)
saved_attributes.append(fbapi_Attribute.AttributeEnd(builder))

# Save attributes vector.
# `PrependUOffsetTRelative` converts each buffer-end-relative offset
# into a 4-byte self-relative pointer (`cursor - target`) and writes
# it backwards (`head` decreases), so `reversed(...)` places element
# 0 at the lowest address right after the vector length header.
fbapi_Example.ExampleStartAttributesVector(builder,
len(saved_attributes))
for offset in reversed(saved_attributes):
builder.PrependUOffsetTRelative(offset)
attributes_vector_offset = builder.EndVector()

# Save the example.
fbapi_Example.ExampleStart(builder)
fbapi_Example.ExampleAddAttributes(builder,
attributes_vector_offset)
examples.append(fbapi_Example.ExampleEnd(builder))

# Save examples vector.
fbapi_Shard.ShardStartExamplesVector(self._builder, len(self._examples))
for offset in reversed(self._examples):
self._builder.PrependUOffsetTRelative(offset)
examples_vector_offset = self._builder.EndVector()
fbapi_Shard.ShardStartExamplesVector(builder, len(examples))
for offset in reversed(examples):
builder.PrependUOffsetTRelative(offset)
examples_vector_offset = builder.EndVector()

# Save the shard.
fbapi_Shard.ShardStart(self._builder)
fbapi_Shard.ShardAddExamples(self._builder, examples_vector_offset)
shard = fbapi_Shard.ShardEnd(self._builder)
fbapi_Shard.ShardStart(builder)
fbapi_Shard.ShardAddExamples(builder, examples_vector_offset)
shard = fbapi_Shard.ShardEnd(builder)

# Finish the builder.
self._builder.Finish(shard)
# Finish the builder (prepends the root pointer to `shard`; `Output()`
# slices the populated tail `builder.Bytes[builder.Head():]`).
builder.Finish(shard)

# Hash check-sums must be computed from already compressed file.
file_content: bytes = CompressedFile(
self.dataset_structure.compression).compress(
bytes(self._builder.Output()))
self._builder = None
self.dataset_structure.compression).compress(bytes(
builder.Output()))
hash_checksums: tuple[str, ...] = hash_checksums_from_bytes(
file_content=file_content,
hashes=self.dataset_structure.hash_checksum_algorithms,
Expand Down
Loading
Loading