diff --git a/src/sedpack/io/shard/shard_writer_base.py b/src/sedpack/io/shard/shard_writer_base.py index 382b0c77..49fa8bf5 100644 --- a/src/sedpack/io/shard/shard_writer_base.py +++ b/src/sedpack/io/shard/shard_writer_base.py @@ -25,7 +25,9 @@ class ShardWriterBase(ABC): - """Shard writing capabilities. + """Base class for shard writing. Buffers examples in memory so the shard + is pickleable and format-specific serialization in `close()` can run in a + different process. """ def __init__(self, dataset_structure: DatasetStructure, @@ -42,9 +44,7 @@ def __init__(self, dataset_structure: DatasetStructure, # Information needed to save the shard. self.dataset_structure: DatasetStructure = dataset_structure self._shard_file: Path = shard_file - - # Make sure that the directory exists. - self._shard_file.parent.mkdir(exist_ok=True, parents=True) + self._examples: list[ExampleT] = [] # Make sure that the compression is supported for this shard file type. assert dataset_structure.compression in self.supported_compressions() @@ -96,17 +96,18 @@ def attribute_value_to_bytes( return np.asarray(value_np, dtype=np.uint8, order="C").tobytes() def write(self, values: ExampleT) -> None: - """Write an example on disk. Writing may be buffered. + """Buffer an example in memory to be serialized on `close()`. Args: values (ExampleT): Attribute values. """ + example: ExampleT = {} # Check the values are correct type and shape. for attribute in self.dataset_structure.saved_data_description: value = values[attribute.name] if attribute.dtype == "bytes": - self.attribute_value_to_bytes( + example[attribute.name] = self.attribute_value_to_bytes( attribute=attribute, value=value, ) @@ -119,30 +120,41 @@ def write(self, values: ExampleT) -> None: # If the attribute dtype is "bytes" and shape is empty tuple we # consider this a variable size attribute and do not check shape. - if attribute.has_variable_size(): - continue + if not attribute.has_variable_size(): + # Else check the shape (the value should be a NumPy array but + # maybe it is an int or bytearray). + current_shape = np.asarray(value).shape + if current_shape != attribute.shape: + raise ValueError( + f"Attribute {attribute.name} has shape " + f"{current_shape} expected {attribute.shape}") - # Else check the shape (the value should be a NumPy array but maybe - # it is an int or bytearray). - current_shape = np.asarray(value).shape - if current_shape != attribute.shape: - raise ValueError(f"Attribute {attribute.name} has shape " - f"{current_shape} expected {attribute.shape}") + example[attribute.name] = (np.copy(value) if isinstance( + value, np.ndarray) else value) - self._write(values=values) + self._examples.append(example) - @abstractmethod - def _write(self, values: ExampleT) -> None: - """Write an example on disk. Writing may be buffered. + def close(self) -> tuple[str, ...]: + """Serialize buffered examples into the shard file(-s) and return hash + checksums. + """ + if not self._examples: + # Nothing to save. + assert not self._shard_file.is_file() + return () - Args: + # Make sure that the directory exists before writing the shard file. + self._shard_file.parent.mkdir(exist_ok=True, parents=True) - values (ExampleT): Attribute values. - """ + hash_checksums = self._close() + self._examples.clear() + assert self._shard_file.is_file() + return hash_checksums @abstractmethod - def close(self) -> tuple[str, ...]: - """Close the shard file(-s). + def _close(self) -> tuple[str, ...]: + """Implementation of shard file(-s) serialization for non-empty + `self._examples`. """ @staticmethod diff --git a/src/sedpack/io/shard/shard_writer_flatbuffer.py b/src/sedpack/io/shard/shard_writer_flatbuffer.py index 1f1ce087..7ec1b2f9 100644 --- a/src/sedpack/io/shard/shard_writer_flatbuffer.py +++ b/src/sedpack/io/shard/shard_writer_flatbuffer.py @@ -24,7 +24,7 @@ from sedpack.io.compress import CompressedFile from sedpack.io.metadata import Attribute, DatasetStructure -from sedpack.io.types import AttributeValueT, CompressionT, ExampleT +from sedpack.io.types import AttributeValueT, CompressionT from sedpack.io.shard.shard_writer_base import ShardWriterBase from sedpack.io.utils import hash_checksums_from_bytes @@ -56,51 +56,6 @@ def __init__(self, dataset_structure: DatasetStructure, shard_file=shard_file, ) - # Offsets. - self._examples: list[int] = [] - - self._builder: Builder | None = None # type: ignore[no-any-unimported] - - def _write(self, values: ExampleT) -> None: - """Write an example on disk. Writing may be buffered. - - Args: - - values (ExampleT): Attribute values. - """ - if self._builder is None: - self._builder = Builder(0) - - # Since we are not saving attribute names we need to make sure to - # iterate in the correct order. - saved_attributes: list[int] = [] - for attribute in self.dataset_structure.saved_data_description: - attribute_bytes: int = self.save_numpy_vector_as_bytearray( - builder=self._builder, - attribute=attribute, - value=values[attribute.name], - ) - - fbapi_Attribute.AttributeStart(self._builder) - fbapi_Attribute.AttributeAddAttributeBytes( - self._builder, - attribute_bytes, - ) - saved_attributes.append(fbapi_Attribute.AttributeEnd(self._builder)) - - # Save attributes vector. - fbapi_Example.ExampleStartAttributesVector(self._builder, - len(saved_attributes)) - for offset in reversed(saved_attributes): - self._builder.PrependUOffsetTRelative(offset) - attributes_vector_offset = self._builder.EndVector() - - # Save the example. - fbapi_Example.ExampleStart(self._builder) - fbapi_Example.ExampleAddAttributes(self._builder, - attributes_vector_offset) - self._examples.append(fbapi_Example.ExampleEnd(self._builder)) - @staticmethod def _np_to_bytes( attribute: Attribute, @@ -205,6 +160,10 @@ def save_numpy_vector_as_bytearray( # type: ignore[no-any-unimported] length: int = len(byte_representation) # Start a vector and move head accordingly. + # FlatBuffers grows `builder.Bytes` backwards from the end toward index + # 0. `StartVector` ensures capacity and aligns the write cursor + # (`builder.Head()`, an index from the start of `builder.Bytes`), so + # decrementing `head` by `length` reserves the slice for a bulk copy. builder.StartVector( elemSize=1, # Storing bytes. numElems=length, @@ -221,11 +180,16 @@ def save_numpy_vector_as_bytearray( # type: ignore[no-any-unimported] builder.vectorNumElems = length # Return the vector offset. + # `EndVector` prepends the 4-byte element count (`vectorNumElems`) and + # returns a `UOffsetT` offset measured from the *end* of `builder.Bytes` + # (`len(builder.Bytes) - builder.Head()`), which remains valid if the + # buffer is later reallocated at the front. return builder.EndVector() # type: ignore[no-any-return] @tenacity.retry( wait=tenacity.wait_random(min=60, max=5 * 60), # 1 to 5 minutes stop=tenacity.stop_after_attempt(10), + retry=tenacity.retry_if_exception_type(OSError), reraise=True, ) def _write_file(self, file_content: bytes) -> None: @@ -233,39 +197,70 @@ def _write_file(self, file_content: bytes) -> None: with open(self._shard_file, "wb") as file: file.write(file_content) - def close(self) -> tuple[str, ...]: - """Close the shard file(-s). + def _close(self) -> tuple[str, ...]: + """Serialize buffered examples into a FlatBuffer shard file. """ - if not self._examples: - # Nothing to save. - assert not self._shard_file.is_file() - return hash_checksums_from_bytes( - file_content=b"", - hashes=self.dataset_structure.hash_checksum_algorithms, - ) + builder = Builder(0) - if not self._builder: - raise ValueError("Attempting to close a closed shard") + # Since we are not saving attribute names we need to make sure to + # iterate in the correct order. + # FlatBuffers forbids nested table/vector construction (`Start...` / + # `End...` cannot be interleaved). Child objects must be built bottom-up + # first (byte vector -> Attribute table -> Attributes vector -> Example + # table -> Examples vector -> Shard table) and referenced via their + # buffer-end-relative `UOffsetT` offsets. + examples: list[int] = [] + for example in self._examples: + saved_attributes: list[int] = [] + for attribute in self.dataset_structure.saved_data_description: + attribute_bytes: int = self.save_numpy_vector_as_bytearray( + builder=builder, + attribute=attribute, + value=example[attribute.name], + ) + fbapi_Attribute.AttributeStart(builder) + fbapi_Attribute.AttributeAddAttributeBytes( + builder, + attribute_bytes, + ) + saved_attributes.append(fbapi_Attribute.AttributeEnd(builder)) + + # Save attributes vector. + # `PrependUOffsetTRelative` converts each buffer-end-relative offset + # into a 4-byte self-relative pointer (`cursor - target`) and writes + # it backwards (`head` decreases), so `reversed(...)` places element + # 0 at the lowest address right after the vector length header. + fbapi_Example.ExampleStartAttributesVector(builder, + len(saved_attributes)) + for offset in reversed(saved_attributes): + builder.PrependUOffsetTRelative(offset) + attributes_vector_offset = builder.EndVector() + + # Save the example. + fbapi_Example.ExampleStart(builder) + fbapi_Example.ExampleAddAttributes(builder, + attributes_vector_offset) + examples.append(fbapi_Example.ExampleEnd(builder)) # Save examples vector. - fbapi_Shard.ShardStartExamplesVector(self._builder, len(self._examples)) - for offset in reversed(self._examples): - self._builder.PrependUOffsetTRelative(offset) - examples_vector_offset = self._builder.EndVector() + fbapi_Shard.ShardStartExamplesVector(builder, len(examples)) + for offset in reversed(examples): + builder.PrependUOffsetTRelative(offset) + examples_vector_offset = builder.EndVector() # Save the shard. - fbapi_Shard.ShardStart(self._builder) - fbapi_Shard.ShardAddExamples(self._builder, examples_vector_offset) - shard = fbapi_Shard.ShardEnd(self._builder) + fbapi_Shard.ShardStart(builder) + fbapi_Shard.ShardAddExamples(builder, examples_vector_offset) + shard = fbapi_Shard.ShardEnd(builder) - # Finish the builder. - self._builder.Finish(shard) + # Finish the builder (prepends the root pointer to `shard`; `Output()` + # slices the populated tail `builder.Bytes[builder.Head():]`). + builder.Finish(shard) # Hash check-sums must be computed from already compressed file. file_content: bytes = CompressedFile( - self.dataset_structure.compression).compress( - bytes(self._builder.Output())) - self._builder = None + self.dataset_structure.compression).compress(bytes( + builder.Output())) hash_checksums: tuple[str, ...] = hash_checksums_from_bytes( file_content=file_content, hashes=self.dataset_structure.hash_checksum_algorithms, diff --git a/src/sedpack/io/shard/shard_writer_np.py b/src/sedpack/io/shard/shard_writer_np.py index 1b2d57b1..142fdac6 100644 --- a/src/sedpack/io/shard/shard_writer_np.py +++ b/src/sedpack/io/shard/shard_writer_np.py @@ -24,7 +24,7 @@ from sedpack.io.metadata import Attribute, DatasetStructure from sedpack.io.shard.shard_writer_base import ShardWriterBase -from sedpack.io.types import AttributeValueT, CompressionT, ExampleT +from sedpack.io.types import AttributeValueT, CompressionT class ShardWriterNP(ShardWriterBase): @@ -49,8 +49,6 @@ def __init__(self, dataset_structure: DatasetStructure, shard_file=shard_file, ) - self._buffer: dict[str, list[AttributeValueT]] = {} - # A prefix such that prepended it creates a new name without collision # with any attribute name. self._counting_prefix: str = "len" + "_" * max( @@ -69,33 +67,23 @@ def _value_to_np( assert isinstance(value, str) return value case _: - return np.copy(value) - - def _write(self, values: ExampleT) -> None: - """Write an example on disk. Writing may be buffered. - - Args: + return np.asarray(value) - values (dict[str, npt.NDArray[np.generic]]): Attribute values. + def _close(self) -> tuple[str, ...]: + """Serialize buffered examples into an NPZ shard file. """ - # Just buffer all values. - if not self._buffer: - self._buffer = {} - + buffer: dict[str, list[AttributeValueT]] = {} for attribute in self.dataset_structure.saved_data_description: name = attribute.name - value = values[name] - if attribute.dtype != "bytes": - current_values = self._buffer.get(name, []) - current_values.append( + buffer[name] = [ self._value_to_np( attribute=attribute, - value=value, - )) - self._buffer[name] = current_values + value=example[name], + ) for example in self._examples + ] else: - # Extend and remember the length. Attributes with dtype "bytes" + # Extend and remember the length. Attributes with dtype "bytes" # may have variable length. Handle this case. We need to avoid # two things: # - Having wrong length of the bytes array and ideally also @@ -104,39 +92,20 @@ def _write(self, values: ExampleT) -> None: # execution when loading malicious dataset. # We prefix the attribute name by `len_?` such that the new name # is unique and tells us the lengths of the byte arrays. - byte_bytes: bytes = self.attribute_value_to_bytes( - attribute=attribute, - value=value, - ) - counts = self._buffer.get(self._counting_prefix + name, [0]) - counts.append(counts[-1] + - len(byte_bytes) # type: ignore[operator] - ) - self._buffer[self._counting_prefix + name] = counts - - byte_list: list[list[int]] - byte_list = self._buffer.get( # type: ignore[assignment] - name, [[]]) - byte_list[0].extend(byte_bytes) - self._buffer[name] = byte_list # type: ignore[assignment] - - def close(self) -> tuple[str, ...]: - """Close the shard file and return hash check-sums. - """ - if not self._buffer: - assert not self._shard_file.is_file() - return () - - # Deal properly with "bytes" attributes. - for attribute in self.dataset_structure.saved_data_description: - if attribute.dtype != "bytes": - continue - self._buffer[attribute.name] = [ - np.array( - self._buffer[attribute.name][0], - dtype=np.uint8, - ) - ] + byte_chunks: list[bytes] = [ + self.attribute_value_to_bytes( + attribute=attribute, + value=example[name], + ) for example in self._examples + ] + counts: list[int] = [0] + for chunk in byte_chunks: + counts.append(counts[-1] + len(chunk)) + buffer[self._counting_prefix + + name] = counts # type: ignore[assignment] + buffer[name] = [ + np.frombuffer(b"".join(byte_chunks), dtype=np.uint8) + ] # Write the buffer into a file. We should not need to allow_pickle while # saving (the default value is True). But on GitHub actions macos-13 @@ -146,12 +115,12 @@ def close(self) -> tuple[str, ...]: case "ZIP": np.savez_compressed( str(self._shard_file), - **self._buffer, # type: ignore[arg-type] + **buffer, # type: ignore[arg-type] ) case "": np.savez( str(self._shard_file), - **self._buffer, # type: ignore[arg-type] + **buffer, # type: ignore[arg-type] ) case _: # Default should never happen since ShardWriterBase checks that @@ -161,8 +130,6 @@ def close(self) -> tuple[str, ...]: f"ShardWriterNP, supported values are: " f"{self.supported_compressions()}") - self._buffer = {} - assert self._shard_file.is_file() return self._compute_file_hash_checksums() @staticmethod diff --git a/src/sedpack/io/shard/shard_writer_tfrec.py b/src/sedpack/io/shard/shard_writer_tfrec.py index 73b34723..18d27a0b 100644 --- a/src/sedpack/io/shard/shard_writer_tfrec.py +++ b/src/sedpack/io/shard/shard_writer_tfrec.py @@ -18,11 +18,10 @@ """ from pathlib import Path -from typing import Any from sedpack.io.metadata import DatasetStructure from sedpack.io.tfrec.tfdata import to_tfrecord -from sedpack.io.types import ExampleT, CompressionT +from sedpack.io.types import CompressionT from sedpack.io.shard.shard_writer_base import ShardWriterBase @@ -48,46 +47,24 @@ def __init__(self, dataset_structure: DatasetStructure, shard_file=shard_file, ) - # Open the tf.io.TFRecordWriter only with the first `write` call. Make - # it None immediately during a call to `close`. - self._tf_shard_writer: Any | None = None - - def _write(self, values: ExampleT) -> None: - """Write an example on disk. Writing may be buffered. - - Args: - - values (ExampleT): Attribute values. + def _close(self) -> tuple[str, ...]: + """Serialize buffered examples into a TFRecord shard file. """ # TensorFlow is an optional dependency. import tensorflow as tf # pylint: disable=import-outside-toplevel - if (self.dataset_structure.compression - not in ShardWriterTFRec.supported_compressions()): - raise ValueError( - f"Unsupported compression {self.dataset_structure.compression}" - " requested for TFRecordWriter, expected " - f"{ShardWriterTFRec.supported_compressions()}") - if not self._tf_shard_writer: - self._tf_shard_writer = tf.io.TFRecordWriter( + with tf.io.TFRecordWriter( str(self._shard_file), self.dataset_structure.compression, # type: ignore[arg-type] - ) + ) as writer: + for example in self._examples: + writer.write( + to_tfrecord( + saved_data_description=self.dataset_structure. + saved_data_description, + values=example, + )) - example = to_tfrecord( - saved_data_description=self.dataset_structure. - saved_data_description, - values=values, - ) - self._tf_shard_writer.write(example) - - def close(self) -> tuple[str, ...]: - """Close the shard file(-s). - """ - if not self._tf_shard_writer: - raise ValueError("Trying to close a shard that was not open") - self._tf_shard_writer.close() - self._tf_shard_writer = None return self._compute_file_hash_checksums() @staticmethod diff --git a/tests/io/shard/test_shard_writer_base.py b/tests/io/shard/test_shard_writer_base.py index e9c88d14..cf7bb953 100644 --- a/tests/io/shard/test_shard_writer_base.py +++ b/tests/io/shard/test_shard_writer_base.py @@ -12,12 +12,103 @@ # See the License for the specific language governing permissions and # limitations under the License. +from pathlib import Path +import pickle from typing import get_args -from sedpack.io.types import ShardFileTypeT -from sedpack.io.shard.get_shard_writer import _SHARD_FILE_TYPE_TO_CLASS +import numpy as np +import pytest +from sedpack.io.metadata import Attribute, DatasetStructure +from sedpack.io.shard.get_shard_writer import ( + _SHARD_FILE_TYPE_TO_CLASS, + get_shard_writer, +) +from sedpack.io.types import CompressionT, ShardFileTypeT +from sedpack.io.utils import is_module_present -def test_all_file_types_supported(): + +def test_all_file_types_supported() -> None: assert set(_SHARD_FILE_TYPE_TO_CLASS.keys()) == set( get_args(ShardFileTypeT)) + + +@pytest.mark.parametrize( + "shard_file_type,compression", + [ + ("fb", "LZ4"), + ("npz", "ZIP"), + ("tfrec", "GZIP"), + ], +) +def test_shard_writer_is_pickleable_and_buffers_examples( + tmp_path: Path, + shard_file_type: ShardFileTypeT, + compression: CompressionT, +) -> None: + if shard_file_type == "tfrec" and not is_module_present("tensorflow"): + pytest.skip("TensorFlow is optional, skip test if not present.") + + dataset_structure = DatasetStructure( + saved_data_description=[ + Attribute(name="a", shape=(4,), dtype="float32"), + ], + shard_file_type=shard_file_type, + compression=compression, + ) + shard_file = tmp_path / f"shard.{shard_file_type}" + writer = get_shard_writer( + dataset_structure=dataset_structure, + shard_file=shard_file, + ) + + arr = np.array([1.0, 2.0, 3.0, 4.0], dtype=np.float32) + writer.write({"a": arr}) + # Mutating the input array after write must not mutate the buffered example. + arr[0] = 999.0 + writer.write({"a": arr}) + + assert len(writer._examples) == 2 + assert not shard_file.is_file() + + # Round-trip through pickle (simulating sending to another process) and + # close the unpickled writer. + unpickled_writer = pickle.loads(pickle.dumps(writer)) + checksums = unpickled_writer.close() + + assert shard_file.is_file() + assert len(checksums) == len(dataset_structure.hash_checksum_algorithms) + assert unpickled_writer._examples == [] + + +@pytest.mark.parametrize( + "shard_file_type,compression", + [ + ("fb", "LZ4"), + ("npz", "ZIP"), + ("tfrec", "GZIP"), + ], +) +def test_close_empty_shard( + tmp_path: Path, + shard_file_type: ShardFileTypeT, + compression: CompressionT, +) -> None: + if shard_file_type == "tfrec" and not is_module_present("tensorflow"): + pytest.skip("TensorFlow is optional, skip test if not present.") + + dataset_structure = DatasetStructure( + saved_data_description=[ + Attribute(name="a", shape=(4,), dtype="float32"), + ], + shard_file_type=shard_file_type, + compression=compression, + ) + shard_file = tmp_path / f"empty_shard.{shard_file_type}" + writer = get_shard_writer( + dataset_structure=dataset_structure, + shard_file=shard_file, + ) + + assert writer.close() == () + assert not shard_file.is_file() diff --git a/tests/io/test_bytes.py b/tests/io/test_bytes.py index 0641b9fa..e7ffa889 100644 --- a/tests/io/test_bytes.py +++ b/tests/io/test_bytes.py @@ -278,3 +278,139 @@ def test_invalid_bytes_attribute_raises( ) with pytest.raises(ValueError): writer.write({"b": invalid_value}) + + +@pytest.mark.parametrize( + "sentence", + [ + "Příliš žluťoučký kůň úpěl ďábelské ódy. " + "PŘÍLIŠ ŽLUŤOUČKÝ KŮŇ ÚPĚL ĎÁBELSKÉ ÓDY.", + "« Dès Noël, où un zéphyr haï me vêt de glaçons würmiens, " + "je dîne d’exquis rôtis de bœuf au kir à l’aÿ d’âge mûr & cætera ! » " + "« DÈS NOËL, OÙ UN ZÉPHYR HAÏ ME VÊT DE GLAÇONS WÜRMIENS, " + "JE DÎNE D’EXQUIS RÔTIS DE BŒUF AU KIR À L’AŸ D’ÂGE MÛR & CÆTERA ! »", + ], +) +@pytest.mark.parametrize( + "shard_file_type,compression", + [ + ("fb", "LZ4"), + ("npz", "ZIP"), + ("tfrec", "GZIP"), + ], +) +def test_utf8_special_characters( + tmp_path: Path, + shard_file_type: ShardFileTypeT, + compression: CompressionT, + sentence: str, +) -> None: + """Verify UTF-8 strings containing special characters (diacritics and + ligatures) round-trip across shard formats and iterators. + """ + if shard_file_type == "tfrec" and not is_module_present("tensorflow"): + pytest.skip("TensorFlow is optional, skip test if not present.") + + sentences = [sentence] + + dataset_structure = DatasetStructure( + saved_data_description=[ + Attribute(name="text", dtype="str", shape=()), + Attribute(name="utf8_bytes", dtype="bytes", shape=()), + ], + compression=compression, + examples_per_shard=2, + shard_file_type=shard_file_type, + ) + dataset = Dataset.create( + path=tmp_path / f"ds_utf8_{shard_file_type}", + metadata=Metadata(description="Test UTF-8 special characters"), + dataset_structure=dataset_structure, + ) + + with dataset.filler(concurrency=2) as filler: + for item in sentences: + filler.write_example( + values={ + "text": item, + "utf8_bytes": item.encode("utf-8"), + }, + split=TRAIN_SPLIT, + ) + + dataset = Dataset(dataset.path) + dataset.check() + + expected_bytes = [s.encode("utf-8") for s in sentences] + expected_text: list[Any] = (expected_bytes + if shard_file_type == "tfrec" else sentences) + + for iterator in [ + dataset.as_numpy_iterator( + split=TRAIN_SPLIT, + shuffle=0, + repeat=False, + ), + dataset.as_numpy_iterator_concurrent( + split=TRAIN_SPLIT, + shuffle=0, + repeat=False, + ), + ]: + seen = list(iterator) + assert [ex["text"] for ex in seen] == expected_text + assert [ex["utf8_bytes"] for ex in seen] == expected_bytes + + if shard_file_type in ("fb", "npz"): + + async def collect_async() -> list[dict[str, Any]]: + return [ + ex async for ex in dataset.as_numpy_iterator_async( + split=TRAIN_SPLIT, + shuffle=0, + repeat=False, + ) + ] + + seen_async = asyncio.run(collect_async()) + assert [ex["text"] for ex in seen_async] == sentences + assert [ex["utf8_bytes"] for ex in seen_async] == expected_bytes + + if shard_file_type == "fb": + seen_rust = list( + dataset.as_numpy_iterator_rust( + split=TRAIN_SPLIT, + shuffle=0, + repeat=False, + )) + assert [ex["text"] for ex in seen_rust] == sentences + assert [ex["utf8_bytes"] for ex in seen_rust] == expected_bytes + + seen_batched_text: list[str] = [] + seen_batched_bytes: list[bytes] = [] + for batch in dataset.as_numpy_iterator_rust_batched( + split=TRAIN_SPLIT, + shuffle=0, + repeat=False, + batch_size=2, + ): + seen_batched_text.extend(batch["text"]) # type: ignore[arg-type] + seen_batched_bytes.extend( + batch["utf8_bytes"]) # type: ignore[arg-type] + assert seen_batched_text == sentences + assert seen_batched_bytes == expected_bytes + + if is_module_present("tensorflow"): + seen_tf_text: list[str] = [] + seen_tf_bytes: list[bytes] = [] + for batch in dataset.as_tfdataset( + split=TRAIN_SPLIT, + shuffle=0, + repeat=False, + batch_size=2, + ): + seen_tf_text.extend( + b.decode("utf-8") for b in batch["text"].numpy().tolist()) + seen_tf_bytes.extend(batch["utf8_bytes"].numpy().tolist()) + assert seen_tf_text == sentences + assert seen_tf_bytes == expected_bytes