diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9e3f5e2daa..12e3b8e566 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -226,6 +226,10 @@ jobs: - run: uv tool install poethepoet - run: uv sync --all-extras - run: cargo install --locked nexgen --version 0.2.8 --features advanced --force + # The Stream package needs the read, read pages and read page methods of a poll, which only + # a nexgen fork has. + - run: cargo install --locked --git https://github.com/moetemp/nexgen --rev 07b565b6d4339dc8a3f67ce2e9e72ba4925d7f9c --features advanced --root "$RUNNER_TEMP/nexgen-stream" nexgen + - run: echo "NEX_GEN_STREAM_BIN=$RUNNER_TEMP/nexgen-stream/bin/nexgen" >> "$GITHUB_ENV" - run: poe build-develop - run: poe gen-protos - name: Check generation unchanged diff --git a/changelog/added/chattering-ledger-otter.md b/changelog/added/chattering-ledger-otter.md new file mode 100644 index 0000000000..1c8c58885f --- /dev/null +++ b/changelog/added/chattering-ledger-otter.md @@ -0,0 +1,3 @@ +Add the experimental Streams API. Client and Activity code create a Stream, append Records to it, read it and close it. Workflow code can't call a Stream in this version. A client takes its Stream Stores in `stream_config=StreamConfig(stores=[...])`, such as `RedisStreamStore` of the `temporalio-streams-redis` package, or `MemoryStreamStore` of the `temporalio-streams-memory` package in tests. +Add `temporalio.contrib.streams.current_producer` and `as_producer`. An Activity can store its Producer in its heartbeat details and pass it to the appends of its retry. The retry's Records then continue the first attempt's epoch, so a reader does not take them as a replacement of the first attempt's Records. +A Workflow outbound interceptor sees a Stream operation only in its operation-specific method, such as `start_append_records`, and not in `start_system_nexus_operation`. An unknown Stream operation raises `ValueError`. diff --git a/pyproject.toml b/pyproject.toml index 6c41cbba54..2942a54a08 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -173,12 +173,17 @@ exclude = [ 'temporalio/api', 'temporalio/bridge/proto', 'temporalio/nexus/system/workflow_service', + 'temporalio/nexus/system/stream_service', ] [[tool.mypy.overrides]] module = "temporalio.nexus.system.workflow_service.*" ignore_errors = true +[[tool.mypy.overrides]] +module = "temporalio.nexus.system.stream_service.*" +ignore_errors = true + [tool.pydocstyle] convention = "google" # https://github.com/PyCQA/pydocstyle/issues/363#issuecomment-625563088 diff --git a/scripts/gen_nexus_system_api.py b/scripts/gen_nexus_system_api.py index 3b16ce1bb3..fdf38fa90c 100644 --- a/scripts/gen_nexus_system_api.py +++ b/scripts/gen_nexus_system_api.py @@ -23,9 +23,15 @@ ) wit_deps_dir = wit_input_dir / "deps" workflow_output_dir = base_dir / "temporalio" / "nexus" / "system" / "workflow_service" +stream_output_dir = base_dir / "temporalio" / "nexus" / "system" / "stream_service" notification_output_dir = base_dir / "temporalio" / "nexus" / "notifications" workflow_init_path = base_dir / "temporalio" / "workflow" / "__init__.py" NEX_GEN_VERSION = "0.2.8" +# The Stream package needs a newer nexgen, which rejects the other WITs. One pin is enough +# again once the api WITs and a nexgen release agree. The read, read pages and read page methods +# of a poll are only in commit 07b565b6 of the fork https://github.com/moetemp/nexgen. So +# NEX_GEN_STREAM_BIN points at a build of that commit. +NEX_GEN_STREAM_VERSION = "0.2.12" proto_files = [ gen_protos.api_proto_dir / "temporal" @@ -36,10 +42,22 @@ gen_protos.api_proto_dir / "temporal" / "api" + / "streamservice" + / "v1" + / "request_response.proto", + gen_protos.api_proto_dir + / "temporal" + / "api" / "notificationservice" / "v1" / "request_response.proto", ] +# The Workflow functions of these generated packages are exported by temporalio.workflow. +# They are in import order, so that sorting the imports keeps them between the markers. +workflow_export_packages = [ + ("temporalio.nexus.system.stream_service.workflow", stream_output_dir / "workflow"), + ("temporalio.nexus.system.workflow_service", workflow_output_dir), +] def nex_gen_command() -> list[str]: @@ -66,6 +84,21 @@ def nex_gen_command() -> list[str]: return ["nexgen"] +def nex_gen_stream_command() -> list[str]: + bin_path = os.environ.get("NEX_GEN_STREAM_BIN") + if not bin_path: + raise RuntimeError( + "NEX_GEN_STREAM_BIN must point at a nexgen build of commit 07b565b6 " + "of https://github.com/moetemp/nexgen" + ) + version = subprocess.check_output([bin_path, "--version"], text=True).strip() + if version != f"nexgen {NEX_GEN_STREAM_VERSION}": + raise RuntimeError( + f"NEX_GEN_STREAM_BIN is {version}, not nexgen {NEX_GEN_STREAM_VERSION}" + ) + return [bin_path] + + def build_descriptor_set(descriptor_path: Path) -> None: subprocess.check_call( [ @@ -85,14 +118,15 @@ def generate_package( output_dir: Path, descriptor_path: Path, *, - native_api: bool, + flags: list[str], ) -> None: - args = [*command, "python", str(wit_input_dir / wit_name), str(wit_deps_dir)] - if native_api: - args.append("--native-api") subprocess.check_call( [ - *args, + *command, + "python", + str(wit_input_dir / wit_name), + str(wit_deps_dir), + *flags, "--system-nexus", "--support-package", "temporalio.nexus.system._support", @@ -132,17 +166,26 @@ def replace_marker_block( def generate_workflow_exports() -> None: - exports = generated_exports(workflow_output_dir / "__init__.py") + package_exports = [ + (module, generated_exports(package_dir / "__init__.py")) + for module, package_dir in workflow_export_packages + ] import_block = [ "# BEGIN GENERATED NEXUS SYSTEM EXPORTS\n", - "from temporalio.nexus.system.workflow_service import (\n", - *[f" {export},\n" for export in exports], - ")\n", + *[ + line + for module, exports in package_exports + for line in [ + f"from {module} import (\n", + *[f" {export},\n" for export in exports], + ")\n", + ] + ], "# END GENERATED NEXUS SYSTEM EXPORTS\n", ] all_block = [ " # BEGIN GENERATED NEXUS SYSTEM __ALL__\n", - *[f' "{export}",\n' for export in exports], + *[f' "{export}",\n' for _, exports in package_exports for export in exports], " # END GENERATED NEXUS SYSTEM __ALL__\n", ] content = workflow_init_path.read_text() @@ -162,6 +205,7 @@ def generate_workflow_exports() -> None: def generate_nexus_system_api() -> None: required_paths = [ wit_input_dir / "workflow-service.wit", + wit_input_dir / "stream-service.wit", wit_input_dir / "notification-service.wit", wit_deps_dir, *proto_files, @@ -173,23 +217,31 @@ def generate_nexus_system_api() -> None: descriptor_path = Path(temp_dir) / "temporal_api.bin" build_descriptor_set(descriptor_path) command = nex_gen_command() - for wit_name, output_dir, native_api in ( - ("workflow-service.wit", workflow_output_dir, True), - ("notification-service.wit", notification_output_dir, False), + stream_command = nex_gen_stream_command() + for wit_name, output_dir, package_command, flags in ( + ("workflow-service.wit", workflow_output_dir, command, ["--native-api"]), + ( + "stream-service.wit", + stream_output_dir, + stream_command, + ["--targets", "workflow,client"], + ), + ("notification-service.wit", notification_output_dir, command, []), ): shutil.rmtree(output_dir, ignore_errors=True) output_dir.parent.mkdir(parents=True, exist_ok=True) generate_package( - command, + package_command, wit_name, output_dir, descriptor_path, - native_api=native_api, + flags=flags, ) workflow_output_dir.parent.joinpath("__init__.py").touch() generate_workflow_exports() format_paths = [ str(workflow_output_dir), + str(stream_output_dir), str(notification_output_dir), str(workflow_init_path), ] diff --git a/scripts/gen_payload_visitor.py b/scripts/gen_payload_visitor.py index ba5ddc112b..eb41d72864 100644 --- a/scripts/gen_payload_visitor.py +++ b/scripts/gen_payload_visitor.py @@ -24,20 +24,30 @@ def discover_system_nexus_roots() -> list[Descriptor]: - module_path = base_dir / "temporalio" / "nexus" / "system" / "workflow_service" - spec = spec_from_file_location( - "temporalio_nexus_system_workflow_service", - module_path / "__init__.py", - submodule_search_locations=[str(module_path)], - ) - if spec is None or spec.loader is None: - raise RuntimeError(f"Cannot load generated system service from {module_path}") - module = module_from_spec(spec) - sys.modules[spec.name] = module - spec.loader.exec_module(module) + services: list[type] = [] + system_path = base_dir / "temporalio" / "nexus" / "system" + for module_path in sorted(p.parent for p in system_path.glob("*/services.py")): + spec = spec_from_file_location( + f"temporalio_nexus_system_{module_path.name}", + module_path / "__init__.py", + submodule_search_locations=[str(module_path)], + ) + if spec is None or spec.loader is None: + raise RuntimeError( + f"Cannot load generated system service from {module_path}" + ) + module = module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + services.extend( + value + for value in vars(module.services).values() + if isinstance(value, type) and nexusrpc.get_service_definition(value) + ) + annotations = [a for service in services for a in get_type_hints(service).values()] roots: list[Descriptor] = [] - for annotation in get_type_hints(module._services.WorkflowService).values(): + for annotation in annotations: if get_origin(annotation) is not nexusrpc.Operation: continue for operation_type in get_args(annotation): diff --git a/temporalio/api/enums/v1/stream_pb2.pyi b/temporalio/api/enums/v1/stream_pb2.pyi index ac57ccd183..7cbd228962 100644 --- a/temporalio/api/enums/v1/stream_pb2.pyi +++ b/temporalio/api/enums/v1/stream_pb2.pyi @@ -92,7 +92,7 @@ class _StreamErrorReasonEnumTypeWrapper( STREAM_ERROR_REASON_NOT_FOUND: _StreamErrorReason.ValueType # 1 """Nobody created the Stream. NOT_FOUND.""" STREAM_ERROR_REASON_EXPIRED: _StreamErrorReason.ValueType # 2 - """The Stream's Retention or its last-write TTL passed. NOT_FOUND.""" + """The Stream's Retention or its Producer idle TTL passed. NOT_FOUND.""" STREAM_ERROR_REASON_CLOSED: _StreamErrorReason.ValueType # 3 """The Stream is closed. FAILED_PRECONDITION.""" STREAM_ERROR_REASON_FENCED: _StreamErrorReason.ValueType # 4 @@ -139,7 +139,7 @@ STREAM_ERROR_REASON_UNSPECIFIED: StreamErrorReason.ValueType # 0 STREAM_ERROR_REASON_NOT_FOUND: StreamErrorReason.ValueType # 1 """Nobody created the Stream. NOT_FOUND.""" STREAM_ERROR_REASON_EXPIRED: StreamErrorReason.ValueType # 2 -"""The Stream's Retention or its last-write TTL passed. NOT_FOUND.""" +"""The Stream's Retention or its Producer idle TTL passed. NOT_FOUND.""" STREAM_ERROR_REASON_CLOSED: StreamErrorReason.ValueType # 3 """The Stream is closed. FAILED_PRECONDITION.""" STREAM_ERROR_REASON_FENCED: StreamErrorReason.ValueType # 4 diff --git a/temporalio/bridge/sdk-core b/temporalio/bridge/sdk-core index c675c32469..7bb86122ba 160000 --- a/temporalio/bridge/sdk-core +++ b/temporalio/bridge/sdk-core @@ -1 +1 @@ -Subproject commit c675c32469af707e1c979a7b808c70b4a0608b82 +Subproject commit 7bb86122bab4ea1efca2295da75fa7751aeaed0c diff --git a/temporalio/client/__init__.py b/temporalio/client/__init__.py index 2eef41a39d..586b3b96cd 100644 --- a/temporalio/client/__init__.py +++ b/temporalio/client/__init__.py @@ -185,6 +185,12 @@ ScheduleUpdate, ScheduleUpdateInput, ) +from ._stream_store import ( + StreamBatchLimits, + StreamConfig, + StreamSelectContext, + StreamStore, +) from ._worker_versioning import ( BuildIdOp, BuildIdOpAddNewCompatible, @@ -350,6 +356,10 @@ "TaskReachabilityType", "CloudOperationsClient", "Plugin", + "StreamBatchLimits", + "StreamConfig", + "StreamSelectContext", + "StreamStore", "Callback", "_ClientImpl", "_apply_headers", diff --git a/temporalio/client/_client.py b/temporalio/client/_client.py index efef5cb528..d42d7d9b0e 100644 --- a/temporalio/client/_client.py +++ b/temporalio/client/_client.py @@ -25,6 +25,7 @@ import temporalio.api.workflowservice.v1 import temporalio.common import temporalio.converter +import temporalio.nexus.system.stream_service.client import temporalio.runtime import temporalio.service import temporalio.workflow @@ -94,6 +95,7 @@ ScheduleBackfill, ScheduleHandle, ) +from ._stream_store import StreamConfig, _refuse_store_plugins from ._worker_versioning import ( BuildIdOp, TaskReachabilityType, @@ -114,7 +116,9 @@ from ._plugin import Plugin -class Client: +class Client( + temporalio.nexus.system.stream_service.client.StreamServiceClientExtension +): """Client for accessing Temporal. Most users will use :py:meth:`connect` to create a client. The @@ -156,6 +160,7 @@ async def connect( grpc_compression: GrpcCompression = GrpcCompression.GZIP, payload_limits: PayloadLimitsConfig = PayloadLimitsConfig(), header_codec_behavior: HeaderCodecBehavior = HeaderCodecBehavior.NO_CODEC, + stream_config: StreamConfig | None = None, ) -> Self: """Connect to a Temporal server. @@ -225,7 +230,18 @@ async def connect( payload_limits: Warning thresholds for outbound payload/memo sizes. Over-threshold fields are logged but still sent. Set a threshold to 0 to disable it. header_codec_behavior: Encoding behavior for headers sent by the client. + stream_config: The Stream Stores of this client and the selector that + picks one for a new Stream, as + :py:class:`temporalio.converter.ExternalStorage` holds drivers and a + selector. A client with it sends each Stream operation to one of its + Stores. This release has no Streams on the server, so with ``None`` a + Stream operation raises + :py:class:`temporalio.exceptions.StreamConfigError`. + + .. warning:: + This API is experimental and subject to change. """ + _refuse_store_plugins(plugins) connect_config = temporalio.service.ConnectConfig( target_host=target_host, api_key=api_key, @@ -262,6 +278,7 @@ def make_lambda( default_workflow_query_reject_condition=default_workflow_query_reject_condition, header_codec_behavior=header_codec_behavior, plugins=plugins, + stream_config=stream_config, ) def __init__( @@ -275,11 +292,13 @@ def __init__( default_workflow_query_reject_condition: None | (temporalio.common.QueryRejectCondition) = None, header_codec_behavior: HeaderCodecBehavior = HeaderCodecBehavior.NO_CODEC, + stream_config: StreamConfig | None = None, ): """Create a Temporal client from a service client. See :py:meth:`connect` for details on the parameters. """ + _refuse_store_plugins(plugins) # Store the config for tracking config = ClientConfig( service_client=service_client, @@ -289,6 +308,7 @@ def __init__( interceptors=interceptors, default_workflow_query_reject_condition=default_workflow_query_reject_condition, header_codec_behavior=header_codec_behavior, + stream_config=stream_config, ) self._initial_config = config.copy() @@ -3029,6 +3049,7 @@ class ClientConnectConfig(TypedDict, total=False): grpc_compression: GrpcCompression payload_limits: PayloadLimitsConfig header_codec_behavior: HeaderCodecBehavior + stream_config: StreamConfig | None class ClientConfig(TypedDict, total=False): @@ -3043,3 +3064,4 @@ class ClientConfig(TypedDict, total=False): temporalio.common.QueryRejectCondition | None ] header_codec_behavior: Required[HeaderCodecBehavior] + stream_config: Required[StreamConfig | None] diff --git a/temporalio/client/_impl.py b/temporalio/client/_impl.py index 72986ffa70..4766126ae8 100644 --- a/temporalio/client/_impl.py +++ b/temporalio/client/_impl.py @@ -33,6 +33,7 @@ import temporalio.exceptions import temporalio.nexus import temporalio.nexus._operation_context +import temporalio.nexus.system from temporalio.activity import ActivityCancellationDetails from temporalio.converter import ( ActivitySerializationContext, @@ -125,6 +126,13 @@ ScheduleUpdate, ScheduleUpdateInput, ) +from ._stream_store import ( + _STREAM_SERVICE, + StreamConfig, + _activity_target, + _call_stream_store, + _request_ref, +) from ._worker_versioning import WorkerBuildIdVersionSets, WorkerTaskReachability from ._workflow import ( WorkflowExecutionAsyncIterator, @@ -1565,6 +1573,18 @@ async def start_nexus_operation( self, input: StartNexusOperationInput ) -> NexusOperationHandle[Any]: """Start a nexus operation and return a handle to it.""" + if ( + temporalio.nexus.system.is_system_endpoint(input.endpoint) + and input.service == _STREAM_SERVICE.name + ): + # This release has no Streams on the server, so a Stream call needs a Store. + stream_config = self._client._config.get("stream_config") + if stream_config is None: + raise temporalio.exceptions.StreamConfigError( + "This client has no Stream Store. Give one in " + "stream_config=StreamConfig(stores=[...])." + ) + return await self._start_stream_store_operation(stream_config, input) nexus_context = temporalio.converter.NexusSerializationContext( endpoint=input.endpoint, service=input.service, @@ -1648,6 +1668,37 @@ async def start_nexus_operation( operation=input.operation, ) + async def _start_stream_store_operation( + self, stream_config: StreamConfig, input: StartNexusOperationInput + ) -> NexusOperationHandle[Any]: + data_converter = self._client.data_converter + serialization_context = temporalio.nexus.system._get_serialization_context( + input.service, input.operation, input.arg + ) + if serialization_context is not None: + data_converter = data_converter.with_context(serialization_context) + payload_converter = temporalio.nexus.system._get_payload_converter( + data_converter.payload_converter, data_converter.failure_converter + ) + # The generated model converts to its proto message with no payload in between. + request = payload_converter._to_transfer_value(input.arg) + store = stream_config._store_for(_request_ref(input.arg), _activity_target()) + response = await _call_stream_store( + store, data_converter, input.operation, request + ) + handle: NexusOperationHandle[Any] = NexusOperationHandle( + self._client, + input.id, + result_type=input.result_type, + endpoint=input.endpoint, + service=input.service, + operation=input.operation, + ) + handle._known_outcome = payload_converter._from_transfer_value( + response, input.result_type + ) + return handle + async def describe_nexus_operation( self, input: DescribeNexusOperationInput ) -> NexusOperationExecutionDescription: diff --git a/temporalio/client/_stream_store.py b/temporalio/client/_stream_store.py new file mode 100644 index 0000000000..6b6d5778e9 --- /dev/null +++ b/temporalio/client/_stream_store.py @@ -0,0 +1,400 @@ +from __future__ import annotations + +import dataclasses +from abc import ABC, abstractmethod +from collections.abc import Callable, Sequence +from typing import Any + +import google.protobuf.message +import nexusrpc + +import temporalio.activity +import temporalio.api.streamservice.v1 +import temporalio.bridge._visitor +import temporalio.converter +import temporalio.nexus.system.stream_service +from temporalio.converter import StorageDriverActivityInfo +from temporalio.exceptions import StreamConfigError + +_MAX_RECORD_BYTES = 1536 * 1024 +_MAX_BATCH_BYTES = 4 * 1024 * 1024 +_MAX_BATCH_RECORDS = 1000 + + +@dataclasses.dataclass(frozen=True) +class StreamBatchLimits: + """The limits of one batch of Records on a Stream Store. + + The defaults are the largest batch that any Store may take. A Store can allow less, and + never more. + + .. warning:: + This API is experimental and subject to change. + """ + + max_record_bytes: int = _MAX_RECORD_BYTES + """The most bytes of one stored Record, from 1 to 1,572,864. The stored form of a Record + has its Producer and its body.""" + + max_batch_bytes: int = _MAX_BATCH_BYTES + """The most bytes of the stored Records of one batch, from 1 to 4,194,304.""" + + max_batch_records: int = _MAX_BATCH_RECORDS + """The most Records of one batch, from 1 to 1,000.""" + + def __post_init__(self) -> None: + """Check that each limit is from 1 to the default of its field. + + Raises: + ValueError: A limit is below 1 or above its default. + """ + for name, most in ( + ("max_record_bytes", _MAX_RECORD_BYTES), + ("max_batch_bytes", _MAX_BATCH_BYTES), + ("max_batch_records", _MAX_BATCH_RECORDS), + ): + value = getattr(self, name) + if not 1 <= value <= most: + raise ValueError(f"{name} must be from 1 to {most}, not {value}") + + +class StreamStore(ABC): + """A Store that answers the Stream operations of a client. + + A client with a Store sends each Stream operation to it. This release has no + Streams on the server, so a client needs a Store for any Stream operation. Use a + Store of a package, such as ``RedisStreamStore`` of ``temporalio-streams-redis``, or + ``MemoryStreamStore`` of ``temporalio-streams-memory`` in tests. + + To keep Streams in another system, write a Store that implements :py:attr:`name` + and the five operations. Each operation takes and gives a message of + ``temporalio.api.streamservice.v1``. The bodies of the Records arrive as payloads + that the SDK already encoded with the data converter and the payload codec, so a + Store keeps payloads and not the user's types. A failed call raises + :py:class:`temporalio.exceptions.StreamError` with a reason. + + .. warning:: + This API is experimental and subject to change. + """ + + @property + @abstractmethod + def name(self) -> str: + """Name of the Store, which the refs of its Streams carry.""" + ... + + @property + def limits(self) -> StreamBatchLimits: + """The limits of one batch on this Store. + + The default is the largest batch that any Store may take. A Store that allows less + overrides this property, and refuses a larger batch itself. + """ + return StreamBatchLimits() + + @abstractmethod + async def create_stream( + self, request: temporalio.api.streamservice.v1.CreateStreamRequest + ) -> temporalio.api.streamservice.v1.CreateStreamResponse: + """Create a Stream, or get the open one when the conflict policy allows it. + + The ref of the response carries the name of this Store. + """ + ... + + @abstractmethod + async def append_records( + self, request: temporalio.api.streamservice.v1.AppendRecordsRequest + ) -> temporalio.api.streamservice.v1.AppendRecordsResponse: + """Append one batch of Records as one Producer. + + A batch that is sent again with the same Producer, epoch and sequence is + written once. + """ + ... + + @abstractmethod + async def read_records( + self, request: temporalio.api.streamservice.v1.ReadRecordsRequest + ) -> temporalio.api.streamservice.v1.ReadRecordsResponse: + """Read one page of Records after an offset. + + At the end of the Stream it waits up to the request's ``wait`` for a new Record. + """ + ... + + @abstractmethod + async def close_stream( + self, request: temporalio.api.streamservice.v1.CloseStreamRequest + ) -> temporalio.api.streamservice.v1.CloseStreamResponse: + """Close a Stream. Readers drain what is left, and later appends fail.""" + ... + + @abstractmethod + async def describe_stream( + self, request: temporalio.api.streamservice.v1.DescribeStreamRequest + ) -> temporalio.api.streamservice.v1.DescribeStreamResponse: + """Give the status and the newest offset of a Stream.""" + ... + + +@dataclasses.dataclass(frozen=True) +class _StreamOperation: + """How a Store answers one operation of the Stream service.""" + + store_method: str + """The name of the :py:class:`StreamStore` method that answers it.""" + + request_type: type[google.protobuf.message.Message] + + response_type: type[google.protobuf.message.Message] + + async def call( + self, store: StreamStore, request: google.protobuf.message.Message + ) -> google.protobuf.message.Message: + """Call the method of ``store`` for this operation and check its answer.""" + return self.check_response( + store, await getattr(store, self.store_method)(request) + ) + + def check_response( + self, store: StreamStore, response: object + ) -> google.protobuf.message.Message: + """Give ``response`` back when it has the response type of this operation.""" + # A Store that someone wrote can give another type, and the error names it here + # rather than in the conversion to the user's type. + if not isinstance(response, self.response_type): + raise TypeError( + f"The Stream Store '{store.name}' answered {self.store_method} with " + f"{type(response).__name__}, not {self.response_type.__name__}" + ) + return response + + +_STREAM_OPERATIONS: dict[str, _StreamOperation] = { + "CreateStream": _StreamOperation( + "create_stream", + temporalio.api.streamservice.v1.CreateStreamRequest, + temporalio.api.streamservice.v1.CreateStreamResponse, + ), + "AppendRecords": _StreamOperation( + "append_records", + temporalio.api.streamservice.v1.AppendRecordsRequest, + temporalio.api.streamservice.v1.AppendRecordsResponse, + ), + "ReadRecords": _StreamOperation( + "read_records", + temporalio.api.streamservice.v1.ReadRecordsRequest, + temporalio.api.streamservice.v1.ReadRecordsResponse, + ), + "CloseStream": _StreamOperation( + "close_stream", + temporalio.api.streamservice.v1.CloseStreamRequest, + temporalio.api.streamservice.v1.CloseStreamResponse, + ), + "DescribeStream": _StreamOperation( + "describe_stream", + temporalio.api.streamservice.v1.DescribeStreamRequest, + temporalio.api.streamservice.v1.DescribeStreamResponse, + ), +} +"""The operations of the Stream service, by their Nexus operation name.""" + + +@dataclasses.dataclass(frozen=True) +class StreamSelectContext: + """Context passed to :py:attr:`StreamConfig.store_selector` calls. + + .. warning:: + This API is experimental and subject to change. + """ + + id: str + """The id of the Stream.""" + + target: StorageDriverActivityInfo | None = None + """The Activity that makes the call, or ``None`` for a plain client.""" + + +@dataclasses.dataclass(frozen=True) +class StreamConfig: + """The Stream Stores of a client, and the rule that picks one for a new Stream. + + A client with this config sends each Stream operation to one of its Stores. + This release has no Streams on the server, so a client without this config + raises :py:class:`temporalio.exceptions.StreamConfigError` for a Stream + operation. It holds Stores and a selector, as + :py:class:`temporalio.converter.ExternalStorage` holds drivers and a selector. + + A ref carries the name of its Store, so a call on a known Stream goes to the + Store of that name. A create can name its Store with ``store=``. A call whose ref + names no Store goes to the Store that :py:attr:`store_selector` returns, or to the + one Store when there is no selector. With more than one Store and no selector, + such a call raises :py:class:`temporalio.exceptions.StreamConfigError`. + + An Activity reaches the Stores of its Worker's client through + :py:func:`temporalio.activity.client`. Workflow code can't call a Stream in this + version. A Stream call of Workflow code raises ``RuntimeError``, which fails the + Workflow Task. + + .. warning:: + This API is experimental and subject to change. + """ + + stores: Sequence[StreamStore] + """At least one Store. Each Store has a unique name. + + A ref carries the :py:attr:`StreamStore.name` of its Store, and a call looks + the Store up by that name. With more than one Store and no + :py:attr:`store_selector`, each call must name its Store. + """ + + store_selector: Callable[[StreamSelectContext], StreamStore] | None = None + """Picks the Store of a call whose ref names no Store. + + It gets a :py:class:`StreamSelectContext` for each such call, and returns one of + the instances in :py:attr:`stores`. When it is ``None`` and there is one Store, + such a call goes to that Store. When it is ``None`` and there is more than one + Store, such a call raises :py:class:`temporalio.exceptions.StreamConfigError`. + + A Stream that code builds from an id only goes through the selector on each + call, so the selector must give the same Store for the same id. + """ + + _store_map: dict[str, StreamStore] = dataclasses.field( + init=False, repr=False, compare=False + ) + """Name-keyed index of :py:attr:`stores`, built at construction time.""" + + def __post_init__(self) -> None: + """Check the Stores and build the name-keyed map of them. + + Raises :py:class:`temporalio.exceptions.StreamConfigError` if there is no + Store, or if two Stores share a name. + """ + if not self.stores: + raise StreamConfigError("StreamConfig.stores must have at least one Store") + store_map: dict[str, StreamStore] = {} + for store in self.stores: + if store.name in store_map: + raise StreamConfigError( + f"StreamConfig.stores has more than one Store named '{store.name}'. " + "Each Store needs a unique name." + ) + store_map[store.name] = store + object.__setattr__(self, "_store_map", store_map) + + def _select_store(self, context: StreamSelectContext) -> StreamStore: + selector = self.store_selector + if selector is None: + if len(self.stores) > 1: + raise StreamConfigError( + "StreamConfig has more than one Store and no store_selector, and " + f"the call on the Stream '{context.id}' names no Store. Name one " + "with store=, or set StreamConfig.store_selector." + ) + return self.stores[0] + store = selector(context) + if ( + not isinstance(store, StreamStore) + or self._store_map.get(store.name) is not store + ): + raise StreamConfigError( + f"The store_selector returned {store!r}, which isn't one of " + "StreamConfig.stores" + ) + return store + + def _get_store_by_name(self, name: str) -> StreamStore: + store = self._store_map.get(name) + if store is None: + raise StreamConfigError( + f"The ref names the Stream Store '{name}', which this client doesn't have" + ) + return store + + def _store_for( + self, + ref: temporalio.nexus.system.stream_service.StreamRef[Any], + target: StorageDriverActivityInfo | None, + ) -> StreamStore: + if ref.store: + return self._get_store_by_name(ref.store) + return self._select_store(StreamSelectContext(id=ref.id, target=target)) + + +def _request_ref( # pyright: ignore[reportUnusedFunction] + request: Any, +) -> temporalio.nexus.system.stream_service.StreamRef[Any]: + # A create calls its ref `stream_ref` and every other request calls it `ref`, so + # the field is found by its type. + if dataclasses.is_dataclass(request) and not isinstance(request, type): + for field in dataclasses.fields(request): + value = getattr(request, field.name) + if isinstance(value, temporalio.nexus.system.stream_service.StreamRef): + return value + raise TypeError(f"The request {type(request).__name__} has no StreamRef") + + +def _activity_target() -> StorageDriverActivityInfo | None: # pyright: ignore[reportUnusedFunction] + # External storage builds the same value from the running Activity. + if not temporalio.activity.in_activity(): + return None + info = temporalio.activity.info() + return StorageDriverActivityInfo( + id=info.activity_id, + type=info.activity_type, + run_id=info.activity_run_id, + namespace=info.namespace, + ) + + +def _refuse_store_plugins( # pyright: ignore[reportUnusedFunction] + plugins: Sequence[object], +) -> None: + for plugin in plugins: + if isinstance(plugin, StreamStore): + raise StreamConfigError( + f"The Stream Store '{plugin.name}' is in plugins. Give it in " + "stream_config=StreamConfig(stores=[...]) instead." + ) + + +def _stream_service() -> nexusrpc.ServiceDefinition: + service = nexusrpc.get_service_definition( + temporalio.nexus.system.stream_service.StreamService + ) + if service is None: + raise RuntimeError("The generated StreamService has no service definition") + return service + + +_STREAM_SERVICE = _stream_service() + + +def _workflow_stream_call_error() -> RuntimeError: # pyright: ignore[reportUnusedFunction] + # Not an `ApplicationError`, so it fails the Workflow Task and not the Workflow. A + # Worker with code that has no Stream call can then finish the Workflow. + return RuntimeError( + "Workflow code can't call a Stream in this version. Call it from an Activity " + "through activity.client()." + ) + + +async def _call_stream_store( # pyright: ignore[reportUnusedFunction] + store: StreamStore, + data_converter: temporalio.converter.DataConverter, + operation: str, + request: google.protobuf.message.Message, +) -> google.protobuf.message.Message: + stream_operation = _STREAM_OPERATIONS[operation] + if not data_converter.payload_codec: + return await stream_operation.call(store, request) + # The worker package imports the client, so a module level import would cycle. + from temporalio.bridge.worker import _Visitor + + visitor = temporalio.bridge._visitor.PayloadVisitor(skip_search_attributes=True) + await visitor.visit(_Visitor(data_converter._encode_payload_sequence), request) + response = await stream_operation.call(store, request) + await visitor.visit(_Visitor(data_converter._decode_payload_sequence), response) + return response diff --git a/temporalio/contrib/streams/README.md b/temporalio/contrib/streams/README.md new file mode 100644 index 0000000000..4c5a8e46a4 --- /dev/null +++ b/temporalio/contrib/streams/README.md @@ -0,0 +1,42 @@ +# Streams helpers + +> This package is experimental, and its API can change. + +`temporalio.contrib.streams` holds helpers that sit on top of the Streams API of the SDK. They +run in a client or an Activity, and take a client `Stream`. + +## `BufferedAppender` + +`BufferedAppender` batches the appends of one Stream. `append` returns immediately, and the +appender writes the buffered Records as one batch. A batch waits at most `max_delay`. + +```python +from temporalio.contrib.streams import BufferedAppender + +async with BufferedAppender(tokens) as appender: + for chunk in chunks: + appender.append(chunk) +``` + +The end of the `async with` block writes the Records that are still in the buffer. + +## A stored Producer + +`current_producer` gives the Producer that an append of the calling code sends when it passes +no `producer=`. An Activity that resumes from its heartbeat details can store it there, and its +retry can write as the same Producer. Heartbeat details give the Producer back as its proto +message, so `as_producer` turns it back into a `Producer`. + +```python +@activity.defn +async def write_parts(ref: StreamRef[str], parts: list[str]) -> None: + details = activity.info().heartbeat_details + producer = as_producer(details[0]) if details else current_producer() + stream = activity.client().get_stream(ref, body_type=str) + for index in range(details[1] if details else 0, len(parts)): + await stream.append(parts[index], producer=producer) + activity.heartbeat(producer, index + 1) +``` + +The retry's Records continue the first attempt's epoch, so a reader does not take them as a +replacement of the first attempt's Records. diff --git a/temporalio/contrib/streams/__init__.py b/temporalio/contrib/streams/__init__.py new file mode 100644 index 0000000000..95318f94d1 --- /dev/null +++ b/temporalio/contrib/streams/__init__.py @@ -0,0 +1,19 @@ +"""Helpers that sit on top of the Streams API. + +:py:class:`BufferedAppender` batches the appends of one client Stream. +:py:func:`current_producer` gives the Producer that an append of the calling code sends by +default, and :py:func:`as_producer` gives back a Producer that code stored. They run in a +client or an Activity. + +.. warning:: + This package is experimental and may change in future versions. +""" + +from temporalio.contrib.streams._appender import BufferedAppender +from temporalio.contrib.streams._producer import as_producer, current_producer + +__all__ = [ + "BufferedAppender", + "as_producer", + "current_producer", +] diff --git a/temporalio/contrib/streams/_appender.py b/temporalio/contrib/streams/_appender.py new file mode 100644 index 0000000000..8625da3d80 --- /dev/null +++ b/temporalio/contrib/streams/_appender.py @@ -0,0 +1,216 @@ +"""A helper that batches the appends of one Stream. + +.. warning:: + This module is experimental and subject to change. +""" + +from __future__ import annotations + +import asyncio +from datetime import timedelta +from types import TracebackType +from typing import Generic, TypeVar + +from typing_extensions import Self + +import temporalio.nexus.system.stream_service.client + +BodyT = TypeVar("BodyT") + +# The Store refuses an append of more Records than this. Many small Records fit under its byte +# limit, but they keep the Store busy for long. +_MAX_BATCH_RECORDS = 1000 + +_DEFAULT_MAX_DELAY = timedelta(milliseconds=100) + + +class BufferedAppender(Generic[BodyT]): + """Batches the appends of one Stream. + + Use it in code that appends often, such as an Activity that streams model tokens. + :py:meth:`append` returns immediately and does no I/O. One write is in flight at a time. + Records appended during a write wait for the next batch, and they keep their order. A batch + holds at most ``max_batch_records`` Records. + + It takes a client Stream, in a client or an Activity. It writes the buffered Records at + most ``max_delay`` after the first of them, or immediately when they fill a batch. + + Use it in ``async with``, which writes the remaining Records at the end of the block, or + await :py:meth:`flush`. Without ``async with`` or a :py:meth:`flush`, the Records still in + the buffer are lost. The end of an Activity attempt doesn't write them. + + ``await flush()`` is the backpressure point. A caller that appends faster than the Store + accepts Records must await it from time to time, so the buffer doesn't grow without bound. + + After a batch fails, the appender writes nothing more, so a reader sees no gap. Each later + :py:meth:`append` and :py:meth:`flush` raises the error of that batch. + + The appender doesn't count bytes, so a batch over the size limit of the Store fails with + the Store's error. + + .. warning:: + This API is experimental and subject to change. + """ + + def __init__( + self, + stream: temporalio.nexus.system.stream_service.client.Stream[BodyT], + *, + max_delay: timedelta | None = None, + max_batch_records: int | None = None, + ) -> None: + """Create an appender. It starts no write until the first append. + + Args: + stream: The client Stream to append to. + max_delay: The longest time that a Record waits in the buffer before its batch + starts to write. It must not be negative. ``None`` means 100 milliseconds. + max_batch_records: The most Records of one batch, from 1 to 1,000. Set it to the + ``max_batch_records`` of the Store's + :py:attr:`temporalio.client.StreamStore.limits` when the Store allows less. + ``None`` means 1,000. + + Raises: + ValueError: ``max_delay`` is negative, or ``max_batch_records`` is below 1 or + above 1,000. + """ + if max_delay is not None and max_delay < timedelta(): + raise ValueError("max_delay must not be negative") + if max_batch_records is None: + max_batch_records = _MAX_BATCH_RECORDS + elif not 1 <= max_batch_records <= _MAX_BATCH_RECORDS: + raise ValueError( + f"max_batch_records must be from 1 to {_MAX_BATCH_RECORDS}, " + f"not {max_batch_records}" + ) + self._stream = stream + self._max_delay = max_delay + self._max_batch_records = max_batch_records + self._buffer: list[BodyT] = [] + # When the oldest buffered Record arrived, so its batch starts at most max_delay later. + self._first_at = 0.0 + self._timer: asyncio.Handle | None = None + self._writing: asyncio.Task[None] | None = None + self._appended = 0 + # Count of the Records that a write took or that a failed batch left, written or not. + self._settled = 0 + self._error: BaseException | None = None + self._last_offset: bytes | None = None + self._unwritten = 0 + + def append(self, *records: BodyT) -> None: + """Add one Record for each value to the buffer, and return immediately. + + A list is one value. + + Raises: + TypeError: No value was given. + temporalio.exceptions.StreamError: A batch failed, so the appender takes no more + Records. + """ + self._raise_error() + if not records: + raise TypeError("append takes one or more values") + loop = asyncio.get_running_loop() + if not self._buffer: + self._first_at = loop.time() + self._buffer.extend(records) + self._appended += len(records) + self._schedule() + + async def flush(self) -> bytes | None: + """Write every Record appended before the call. + + Returns: + The ``last_offset`` of the last written batch, or ``None`` if the appender has + written nothing. + + Raises: + temporalio.exceptions.StreamError: A batch failed. + """ + target = self._appended + while self._settled < target: + if self._writing is None: + self._start() + assert self._writing is not None + # The write goes on when the caller is cancelled, so no batch is left half done. + await asyncio.shield(self._writing) + self._raise_error() + return self._last_offset + + @property + def unwritten(self) -> int: + """How many Records the failed batch and the ones after it left unwritten.""" + return self._unwritten + + async def __aenter__(self) -> Self: + """Use the appender in ``async with``.""" + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + tb: TracebackType | None, + ) -> None: + """Write the buffered Records. A write error doesn't hide an error of the block.""" + try: + await self.flush() + except Exception: + if exc is None: + raise + + def _raise_error(self) -> None: + if self._error is not None: + raise self._error + + def _schedule(self) -> None: + if self._writing is not None or not self._buffer: + return + if len(self._buffer) >= self._max_batch_records: + self._start() + elif self._timer is None: + max_delay = self._max_delay or _DEFAULT_MAX_DELAY + self._timer = asyncio.get_running_loop().call_at( + self._first_at + max_delay.total_seconds(), self._start + ) + + def _start(self) -> None: + if self._timer is not None: + self._timer.cancel() + self._timer = None + if self._writing is not None or not self._buffer: + return + batch = self._buffer[: self._max_batch_records] + del self._buffer[: self._max_batch_records] + self._writing = asyncio.get_running_loop().create_task(self._write(batch)) + + async def _write(self, batch: list[BodyT]) -> None: + try: + # The Store numbers each batch, since the appender sends no `sequence` and never + # sends a batch twice. A `sequence` counts for a Producer on a Stream, and two + # appenders of one Activity attempt would each start at 1. + response = await self._stream.append(*batch) + self._last_offset = response.last_offset + except asyncio.CancelledError: + # A later call must not report a cancel that its caller didn't ask for. + self._fail(RuntimeError("The write of a batch was cancelled"), len(batch)) + raise + except Exception as err: + self._fail(err, len(batch)) + finally: + self._settled += len(batch) + self._writing = None + # Schedule the Records appended during the write, for when their delay ends or + # they fill a batch. + self._schedule() + + def _fail(self, error: BaseException, count: int) -> None: + # The Records after a failed batch are never written, so a reader sees no gap. + self._error = error + self._unwritten += count + len(self._buffer) + self._settled += len(self._buffer) + self._buffer.clear() + if self._timer is not None: + self._timer.cancel() + self._timer = None diff --git a/temporalio/contrib/streams/_producer.py b/temporalio/contrib/streams/_producer.py new file mode 100644 index 0000000000..51518d942c --- /dev/null +++ b/temporalio/contrib/streams/_producer.py @@ -0,0 +1,65 @@ +"""The Producer of the calling code, and a Producer from its stored form.""" + +from __future__ import annotations + +import temporalio.api.stream.v1 +import temporalio.client._stream_store +import temporalio.converter._payload_converter +import temporalio.nexus.system._support +import temporalio.workflow +from temporalio.nexus.system.stream_service import Producer + + +def current_producer() -> Producer: + """Give the Producer that an append or a close of the calling code sends by default. + + An ``append`` or a ``close`` with no ``producer=`` sends this Producer. In an + Activity or a Local Activity it is the Producer of the current attempt. In any other + code it is the client Producer of this process. It does no I/O. + + An Activity can keep the result and pass it to a later ``append`` or ``close`` with + ``producer=``. That call then writes as this Producer with its epoch, also in a + later attempt. Heartbeat details give it back as its proto message, so pass it + through :py:func:`as_producer` first. + + Raises: + RuntimeError: It runs in Workflow code, which can't call a Stream in this + version. + + .. warning:: + This API is experimental and subject to change. + """ + if temporalio.workflow.in_workflow(): + raise temporalio.client._stream_store._workflow_stream_call_error() + return temporalio.nexus.system._support.default_producer() + + +def as_producer(value: object) -> Producer: + """Give a Producer for a Producer or for its proto message. + + A Producer that went through a data converter with no type hint, such as one in + ``activity.info().heartbeat_details``, comes back as the proto message + ``temporalio.api.stream.v1.Producer``. This function turns it back into the + :py:class:`temporalio.nexus.system.stream_service.Producer` that ``append`` and + ``close`` take in ``producer=``. Such a Producer comes back as it is. + + Raises: + TypeError: The value is neither a Producer nor its proto message. + + .. warning:: + This API is experimental and subject to change. + """ + if isinstance(value, Producer): + return value + if isinstance(value, temporalio.api.stream.v1.Producer): + converter = ( + temporalio.converter._payload_converter._get_transfer_type_converter( + Producer + ) + ) + if converter is None: + raise TypeError("The generated Producer has no proto transfer type") + return converter.from_transfer_type(value, Producer) + raise TypeError( + f"as_producer takes a Producer or its proto message, not {type(value).__name__}" + ) diff --git a/temporalio/exceptions.py b/temporalio/exceptions.py index 03a623c21e..3f2df0fc9a 100644 --- a/temporalio/exceptions.py +++ b/temporalio/exceptions.py @@ -8,6 +8,7 @@ import temporalio.api.enums.v1 import temporalio.api.failure.v1 +import temporalio.api.stream.v1 class TemporalError(Exception): @@ -198,6 +199,76 @@ def category(self) -> ApplicationErrorCategory: return self._category +_STREAM_ERROR_REASON_UNSPECIFIED = ( + temporalio.api.enums.v1.StreamErrorReason.STREAM_ERROR_REASON_UNSPECIFIED +) + +# The one reason that a retry can fix. +_STREAM_RETRYABLE_REASONS: frozenset[int] = frozenset( + {temporalio.api.enums.v1.StreamErrorReason.STREAM_ERROR_REASON_STORAGE_UNAVAILABLE} +) + + +class StreamError(ApplicationError): + """A failed Stream call. ``reason`` tells the cases apart. + + One class covers every failed call in client and Activity code, so match on + ``reason`` and not on the type. The reason is a ``StreamErrorReason``, such as + ``STREAM_ERROR_REASON_NOT_FOUND`` (nobody created the Stream), + ``STREAM_ERROR_REASON_CLOSED`` (the Stream is closed), or + ``STREAM_ERROR_REASON_ALREADY_EXISTS`` (a strict create of a Stream that exists). + A Store outage, ``STREAM_ERROR_REASON_STORAGE_UNAVAILABLE``, is the one retryable + reason. + + .. warning:: + This API is experimental and subject to change. + """ + + def __init__( + self, + message: str, + *, + reason: temporalio.api.enums.v1.StreamErrorReason.ValueType | None = None, + ) -> None: + """Initialize a Stream error. ``reason`` defaults to the unspecified reason.""" + if reason is None: + reason = _STREAM_ERROR_REASON_UNSPECIFIED + # The details carry the reason, so the error keeps it when it crosses an + # Activity or a Nexus operation. + super().__init__( + message, + temporalio.api.stream.v1.StreamErrorDetails(reason=reason), + type="StreamError", + non_retryable=reason not in _STREAM_RETRYABLE_REASONS, + ) + self._reason = reason + + @property + def reason(self) -> temporalio.api.enums.v1.StreamErrorReason.ValueType: + """Why the Stream call failed.""" + return self._reason + + +class StreamConfigError(RuntimeError): + """A Stream call can't find its Store, because of how the Stores are set up. + + It is not a :py:class:`StreamError`. A Store didn't fail. The setup of the Stores + is wrong, and a retry won't help. These are its cases. + + * A Stream call is made on a client with no Stream config. This release has no + Streams on the server, so every Stream call needs a Store. + * A :py:class:`temporalio.client.StreamConfig` has no Store, or has two Stores + with one name. + * A call names no Store on a config with more than one Store and no selector. + * A ref names a Store that the config of the client doesn't have. + * The selector returns a Store that isn't one of the config's Stores. + * A :py:class:`temporalio.client.StreamStore` is given in ``plugins``. + + .. warning:: + This API is experimental and subject to change. + """ + + class CancelledError(FailureError): """Error raised on workflow/activity cancellation.""" diff --git a/temporalio/nexus/system/__init__.py b/temporalio/nexus/system/__init__.py index 1f8f7c6d29..d3096b1a00 100644 --- a/temporalio/nexus/system/__init__.py +++ b/temporalio/nexus/system/__init__.py @@ -19,6 +19,7 @@ from temporalio.bridge._visitor_functions import VisitorFunctions from temporalio.converter import BinaryProtoPayloadConverter, CompositePayloadConverter from temporalio.converter._payload_converter import ( + _get_transfer_type_converter, _TemporalTransferTypePayloadConverter, ) @@ -123,6 +124,27 @@ def from_payloads( with _user_converter_context(self._user_converters): return self._outer_payload_converter.from_payloads(payloads, type_hints) + def _to_transfer_value(self, value: Any) -> Any: + """Convert a system Nexus model to its proto message, with no payload in between.""" + converter = _get_transfer_type_converter(type(value)) + if converter is None: + raise TypeError(f"{type(value).__name__} has no transfer type") + with _user_converter_context(self._user_converters): + return converter.to_transfer_type(value) + + def _from_transfer_value(self, value: Any, type_hint: type | None) -> Any: + """Convert a proto message to the system Nexus model of ``type_hint``. + + With no ``type_hint``, or one that has no transfer type, it gives the message. + """ + converter = ( + _get_transfer_type_converter(type_hint) if type_hint is not None else None + ) + if converter is None or type_hint is None: + return value + with _user_converter_context(self._user_converters): + return converter.from_transfer_type(value, type_hint) + def is_system_endpoint(endpoint: str) -> bool: """Return whether a Nexus endpoint is the Temporal system endpoint. @@ -171,7 +193,7 @@ async def maybe_visit_payload( def _get_payload_converter( # pyright: ignore[reportUnusedFunction] user_payload_converter: temporalio.converter.PayloadConverter, user_failure_converter: temporalio.converter.FailureConverter, -) -> temporalio.converter.PayloadConverter: +) -> _SystemNexusPayloadConverter: """Return the fixed payload converter for system Nexus outer envelopes.""" return _SystemNexusPayloadConverter(user_payload_converter, user_failure_converter) @@ -182,9 +204,14 @@ def _get_serialization_context( # pyright: ignore[reportUnusedFunction] request: Any, ) -> temporalio.converter.SerializationContext | None: """Return the serialization context for a system Nexus operation.""" + from .stream_service.workflow import ( + __nexus_operation_registry__ as stream_operation_registry, + ) from .workflow_service import __nexus_operation_registry__ - operation_info = __nexus_operation_registry__.get((service, operation)) + operation_info = __nexus_operation_registry__.get( + (service, operation) + ) or stream_operation_registry.get((service, operation)) if operation_info is None or operation_info.serialization_context is None: return None return operation_info.serialization_context(request) @@ -194,3 +221,10 @@ def _get_serialization_context( # pyright: ignore[reportUnusedFunction] "TEMPORAL_SYSTEM_ENDPOINT", "is_system_endpoint", ] + +# The generated Workflow modules of `stream_service` import `temporalio.workflow` at their top, +# and `temporalio.workflow` imports `create_stream` and `get_stream` from them. So this package +# loads `temporalio.workflow` as its last statement, when every name above exists. Then +# `temporalio.workflow` is complete before a generated Workflow module starts, whichever module +# is the first import. +import temporalio.workflow # noqa: E402 # pyright: ignore[reportUnusedImport] diff --git a/temporalio/nexus/system/_support.py b/temporalio/nexus/system/_support.py index 15a8e80102..55f53b59a1 100644 --- a/temporalio/nexus/system/_support.py +++ b/temporalio/nexus/system/_support.py @@ -1,9 +1,15 @@ import collections.abc +import dataclasses +import functools import typing -from datetime import timedelta +import urllib.parse +import uuid +from datetime import datetime, timedelta import google.protobuf.duration_pb2 +import google.protobuf.timestamp_pb2 +import temporalio.activity import temporalio.api.common.v1.message_pb2 as common_pb2 import temporalio.api.enums.v1.workflow_pb2 as workflow_enums_pb2 import temporalio.api.failure.v1.message_pb2 as failure_pb2 @@ -13,12 +19,157 @@ import temporalio.converter import temporalio.nexus.system +if typing.TYPE_CHECKING: + from temporalio.nexus.system.stream_service.models import ( + Producer, + ReadRecordsRequest, + ReadRecordsResponse, + StreamRecord, + ) + +_BodyT = typing.TypeVar("_BodyT") + class SignalWithStartWorkflowModelRequest(typing.Protocol): namespace: str id: str +def request_id() -> str: + import temporalio.workflow + + # Each run of a chain sends the same create, and the Store answers a create with the + # request id that made the Stream as a success. One id for the chain also replays. + if temporalio.workflow.in_workflow(): + return temporalio.workflow.info().first_execution_run_id + return str(uuid.uuid4()) + + +def _producer_id(*parts: str) -> str: + # Quoting keeps a ':' inside an id from shifting the parts. + return ":".join(urllib.parse.quote(part, safe="") for part in parts) + + +# One id for the process, because nothing fences a client write. +@functools.cache +def _client_producer_id() -> str: + return _producer_id("client", str(uuid.uuid4())) + + +def _timestamp(value: datetime) -> google.protobuf.timestamp_pb2.Timestamp: + proto = google.protobuf.timestamp_pb2.Timestamp() + proto.FromDatetime(value) + return proto + + +def default_producer() -> "Producer": + from temporalio.nexus.system.stream_service.models import ( + ActivityEpoch, + ActivityProducer, + ClientProducer, + LocalActivityEpoch, + LocalActivityProducer, + Producer, + ProducerKindClient, + ProducerKindLocalActivity, + ProducerKindStandaloneActivity, + ProducerKindWorkflowActivity, + Timestamp, + WorkflowTaskEpoch, + ) + + if not temporalio.activity.in_activity(): + return Producer( + id=_client_producer_id(), kind=ProducerKindClient(ClientProducer()) + ) + info = temporalio.activity.info() + if info.is_local: + task = info.workflow_task + if task is None or info.workflow_run_id is None: + raise RuntimeError( + "A Stream write from a Local Activity needs its Workflow Task, and the " + "Worker's Core gave none" + ) + started_time = None + if task.started_time is not None: + proto_time = _timestamp(task.started_time) + started_time = Timestamp(seconds=proto_time.seconds, nanos=proto_time.nanos) + # The activity id of a Local Activity is its sequence number in the run, so a + # retried Workflow Task writes as the same Producer with a higher epoch. + return Producer( + id=_producer_id("la", info.workflow_run_id, info.activity_id), + kind=ProducerKindLocalActivity( + LocalActivityProducer( + run_id=info.workflow_run_id, + epoch=LocalActivityEpoch( + task=WorkflowTaskEpoch( + started_event_id=task.started_event_id, + attempt=task.attempt, + started_time=started_time, + ), + attempt=info.attempt, + ), + ) + ), + ) + attempt_time = _timestamp(info.current_attempt_scheduled_time) + epoch = ActivityEpoch( + attempt_scheduled_time=Timestamp( + seconds=attempt_time.seconds, nanos=attempt_time.nanos + ), + attempt=info.attempt, + ) + if info.workflow_run_id is not None: + # The scheduled time stays the same across attempts, and it tells apart two uses + # of one activity id in a run. + schedule_key = str(_timestamp(info.scheduled_time).ToNanoseconds()) + return Producer( + id=_producer_id( + "act", info.workflow_run_id, info.activity_id, schedule_key + ), + kind=ProducerKindWorkflowActivity( + ActivityProducer(run_id=info.workflow_run_id, epoch=epoch) + ), + ) + if info.activity_run_id is None: + raise RuntimeError("A Standalone Activity has no activity run id") + return Producer( + id=_producer_id("sa", info.activity_id, info.activity_run_id), + kind=ProducerKindStandaloneActivity(ActivityProducer(epoch=epoch)), + ) + + +def workflow_producer() -> "Producer": + import temporalio.workflow + from temporalio.nexus.system.stream_service.models import ( + Producer, + ProducerKindWorkflow, + WorkflowProducer, + ) + + run_id = temporalio.workflow.info().run_id + return Producer( + id=_producer_id("wf", run_id), + kind=ProducerKindWorkflow(WorkflowProducer(run_id=run_id)), + ) + + +def next_read_request( + request: "ReadRecordsRequest[_BodyT]", response: "ReadRecordsResponse[_BodyT]" +) -> "ReadRecordsRequest[_BodyT] | None": + if response.done or (request.wait is None and not response.records): + return None + if response.last_offset is None: + return request + return dataclasses.replace(request, from_offset=response.last_offset) + + +def resume_read_request( + request: "ReadRecordsRequest[_BodyT]", item: "StreamRecord[_BodyT]" +) -> "ReadRecordsRequest[_BodyT]": + return dataclasses.replace(request, from_offset=item.offset) + + def retry_policy_from_proto( proto: common_pb2.RetryPolicy, ) -> temporalio.common.RetryPolicy: diff --git a/temporalio/testing/_workflow.py b/temporalio/testing/_workflow.py index 6c8354162c..54add4e363 100644 --- a/temporalio/testing/_workflow.py +++ b/temporalio/testing/_workflow.py @@ -102,6 +102,7 @@ async def start_local( dev_server_download_ttl: timedelta | None = None, dev_server_startup_timeout: timedelta | None = None, ui_port: int | None = None, + stream_config: temporalio.client.StreamConfig | None = None, ) -> WorkflowEnvironment: """Start a full Temporal server locally, downloading if necessary. @@ -165,6 +166,8 @@ async def start_local( become ready after spawning. Defaults to 5 seconds and does not include download time. ui_port: UI port to use if UI is enabled. + stream_config: See parameter of the same name on + :py:meth:`temporalio.client.Client.connect`. Returns: The started CLI dev server workflow environment. @@ -194,6 +197,7 @@ async def start_local( dev_server_download_ttl=dev_server_download_ttl, dev_server_startup_timeout=dev_server_startup_timeout, ui_port=ui_port, + stream_config=stream_config, ) @classmethod @@ -216,6 +220,7 @@ async def start_time_skipping( test_server_extra_args: Sequence[str] = [], test_server_download_ttl: timedelta | None = None, test_server_startup_timeout: timedelta | None = None, + stream_config: temporalio.client.StreamConfig | None = None, ) -> WorkflowEnvironment: """Start a V1 time skipping workflow environment. @@ -276,6 +281,8 @@ async def start_time_skipping( test_server_startup_timeout: Maximum time to wait for the test server to become ready after spawning. Defaults to 5 seconds and does not include download time. + stream_config: See parameter of the same name on + :py:meth:`temporalio.client.Client.connect`. Returns: The started workflow environment with time skipping. @@ -296,6 +303,7 @@ async def start_time_skipping( test_server_extra_args=test_server_extra_args, test_server_download_ttl=test_server_download_ttl, test_server_startup_timeout=test_server_startup_timeout, + stream_config=stream_config, ) @classmethod @@ -582,6 +590,7 @@ async def _bootstrap_dev_server( dev_server_download_ttl: timedelta | None = None, dev_server_startup_timeout: timedelta | None = None, ui_port: int | None = None, + stream_config: temporalio.client.StreamConfig | None = None, ) -> tuple[ temporalio.bridge.testing.EphemeralServer, temporalio.client.Client, @@ -649,6 +658,7 @@ async def _bootstrap_dev_server( rpc_metadata=rpc_metadata, identity=identity, runtime=runtime, + stream_config=stream_config, ) except: try: @@ -719,6 +729,7 @@ async def _create( test_server_extra_args: Sequence[str], test_server_download_ttl: timedelta | None, test_server_startup_timeout: timedelta | None, + stream_config: temporalio.client.StreamConfig | None, ) -> _V1WorkflowEnvironment: runtime = runtime or temporalio.runtime.Runtime.default() download_ttl_ms = None @@ -753,6 +764,7 @@ async def _create( rpc_metadata=rpc_metadata, identity=identity, runtime=runtime, + stream_config=stream_config, ) except: try: diff --git a/temporalio/worker/_interceptor.py b/temporalio/worker/_interceptor.py index 6196f5efe1..d5d49e1ba9 100644 --- a/temporalio/worker/_interceptor.py +++ b/temporalio/worker/_interceptor.py @@ -21,6 +21,9 @@ import temporalio.nexus import temporalio.nexus._util import temporalio.workflow +from temporalio.nexus.system.stream_service.workflow._system_nexus_interceptor import ( + _SystemNexusWorkflowOutboundInterceptorBase as _StreamSystemNexusInterceptorBase, +) from temporalio.nexus.system.workflow_service._system_nexus_interceptor import ( _SystemNexusWorkflowOutboundInterceptorBase, ) @@ -455,7 +458,9 @@ async def handle_update_handler(self, input: HandleUpdateInput) -> Any: return await self.next.handle_update_handler(input) -class WorkflowOutboundInterceptor(_SystemNexusWorkflowOutboundInterceptorBase): +class WorkflowOutboundInterceptor( + _SystemNexusWorkflowOutboundInterceptorBase, _StreamSystemNexusInterceptorBase +): """Outbound interceptor to wrap calls made from within workflows. This should be extended by any workflow outbound interceptors. @@ -472,9 +477,7 @@ def __init__(self, next: WorkflowOutboundInterceptor) -> None: """ self.next = next - def _next_system_nexus_interceptor( - self, - ) -> _SystemNexusWorkflowOutboundInterceptorBase: + def _next_system_nexus_interceptor(self) -> WorkflowOutboundInterceptor: return self.next def continue_as_new(self, input: ContinueAsNewInput) -> NoReturn: @@ -532,7 +535,11 @@ async def start_nexus_operation( async def start_system_nexus_operation( self, input: StartNexusOperationInput[InputT, OutputT] ) -> temporalio.workflow.NexusOperationHandle[OutputT]: - """Intercept a Temporal System Nexus operation started by a workflow.""" + """Intercept a Temporal System Nexus operation started by a workflow. + + A Stream operation does not reach this method. Its operation-specific method, such as + ``start_append_records``, intercepts it. + """ return await self.next.start_system_nexus_operation(input) diff --git a/temporalio/worker/_workflow_instance.py b/temporalio/worker/_workflow_instance.py index 03463e2c03..b4629a293a 100644 --- a/temporalio/worker/_workflow_instance.py +++ b/temporalio/worker/_workflow_instance.py @@ -59,7 +59,17 @@ import temporalio.exceptions import temporalio.nexus.system import temporalio.workflow +from temporalio.client._stream_store import ( + _STREAM_SERVICE, + _workflow_stream_call_error, +) from temporalio.converter import StorageDriverStoreContext, StorageDriverWorkflowInfo +from temporalio.nexus.system.stream_service.workflow._system_nexus_interceptor import ( + _start_system_nexus_operation as _start_stream_system_nexus_operation, +) +from temporalio.nexus.system.stream_service.workflow._system_nexus_interceptor import ( + _SystemNexusWorkflowOutboundInterceptorTerminal as _StreamSystemNexusInterceptorTerminal, +) from temporalio.nexus.system.workflow_service._system_nexus_interceptor import ( _start_system_nexus_operation, _SystemNexusWorkflowOutboundInterceptorTerminal, @@ -1830,23 +1840,25 @@ async def workflow_start_nexus_operation( event_groups: Sequence[temporalio.workflow.EventGroup] | None = None, ) -> temporalio.workflow.NexusOperationHandle[OutputT]: if temporalio.nexus.system.is_system_endpoint(endpoint): - return await _start_system_nexus_operation( - self._outbound, - StartNexusOperationInput( - endpoint=temporalio.nexus.system.TEMPORAL_SYSTEM_ENDPOINT, - service=service, - operation=operation, - input=input, - output_type=output_type, - schedule_to_close_timeout=schedule_to_close_timeout, - schedule_to_start_timeout=schedule_to_start_timeout, - start_to_close_timeout=start_to_close_timeout, - cancellation_type=cancellation_type, - headers=None, - summary=summary, - event_groups=event_groups, - ), + system_input = StartNexusOperationInput( + endpoint=temporalio.nexus.system.TEMPORAL_SYSTEM_ENDPOINT, + service=service, + operation=operation, + input=input, + output_type=output_type, + schedule_to_close_timeout=schedule_to_close_timeout, + schedule_to_start_timeout=schedule_to_start_timeout, + start_to_close_timeout=start_to_close_timeout, + cancellation_type=cancellation_type, + headers=None, + summary=summary, + event_groups=event_groups, ) + if service == _STREAM_SERVICE.name: + return await _start_stream_system_nexus_operation( + self._outbound, system_input + ) + return await _start_system_nexus_operation(self._outbound, system_input) return await self._outbound.start_nexus_operation( StartNexusOperationInput( endpoint=endpoint, @@ -2288,6 +2300,7 @@ async def _outbound_start_nexus_operation( # examples of the evolution of the resulting handle state machine in the sync and async # Nexus response cases. handle: _NexusOperationHandle[OutputT] + payload_converter: temporalio.converter.PayloadConverter async def operation_handle_fn() -> OutputT: return cast( @@ -2355,7 +2368,11 @@ async def _intercept_system_nexus_operation( async def _schedule_system_nexus_operation( self, input: StartNexusOperationInput[Any, OutputT] - ) -> _NexusOperationHandle[OutputT]: + ) -> temporalio.workflow.NexusOperationHandle[OutputT]: + # The server has no Streams, and Workflow code reaches no Store. So a Stream call + # stops here, after the interceptors saw it. + if input.service == _STREAM_SERVICE.name: + raise _workflow_stream_call_error() return await self._outbound_start_nexus_operation(input) #### Miscellaneous helpers #### @@ -3360,7 +3377,9 @@ async def handle_update_handler(self, input: HandleUpdateInput) -> Any: class _WorkflowOutboundImpl( - _SystemNexusWorkflowOutboundInterceptorTerminal, WorkflowOutboundInterceptor + _SystemNexusWorkflowOutboundInterceptorTerminal, + _StreamSystemNexusInterceptorTerminal, + WorkflowOutboundInterceptor, ): def __init__(self, instance: _WorkflowInstanceImpl) -> None: # type: ignore # We are intentionally not calling the base class's __init__ here @@ -3407,7 +3426,12 @@ async def _intercept_system_nexus_operation( async def start_system_nexus_operation( self, input: StartNexusOperationInput[Any, OutputT] - ) -> _NexusOperationHandle[OutputT]: + ) -> temporalio.workflow.NexusOperationHandle[OutputT]: + return await self._instance._schedule_system_nexus_operation(input) + + async def _schedule_system_nexus_operation( + self, input: StartNexusOperationInput[InputT, OutputT] + ) -> temporalio.workflow.NexusOperationHandle[OutputT]: return await self._instance._schedule_system_nexus_operation(input) def start_local_activity( diff --git a/tests/contrib/streams/__init__.py b/tests/contrib/streams/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/contrib/streams/test_appender.py b/tests/contrib/streams/test_appender.py new file mode 100644 index 0000000000..9458ed217a --- /dev/null +++ b/tests/contrib/streams/test_appender.py @@ -0,0 +1,184 @@ +from __future__ import annotations + +import asyncio +import uuid +from datetime import timedelta + +import pytest +from google.protobuf.message import Message + +import temporalio.api.streamservice.v1 +from temporalio.api.enums.v1 import StreamErrorReason +from temporalio.client import Client +from temporalio.contrib.streams import BufferedAppender +from temporalio.nexus.system.stream_service import StreamRef +from temporalio.nexus.system.stream_service.client import Stream +from tests.helpers.streams import DictStreamStore, RecordingStreamStore +from tests.nexus.test_stream_store import _create_stream, _stream_error, _with_store + +# Long enough that only a flush, a full batch or the end of `async with` starts a write. +_NEVER = timedelta(hours=1) + + +class RecordingStore(RecordingStreamStore): + def __init__(self) -> None: + super().__init__(DictStreamStore()) + self.batches: list[int] = [] + self.append_started = asyncio.Event() + self.release: asyncio.Event | None = None + + async def before_call(self, operation: str, request: Message) -> None: + if isinstance(request, temporalio.api.streamservice.v1.AppendRecordsRequest): + self.batches.append(len(request.records)) + self.append_started.set() + if self.release is not None: + await self.release.wait() + + +async def _bodies(stream: Stream[str]) -> list[str]: + return [record.body async for record in stream.read()] + + +async def test_appends_inside_one_delay_go_as_one_batch(client: Client): + store = RecordingStore() + stream = await _create_stream(client, store) + appender = BufferedAppender(stream, max_delay=_NEVER) + appender.append("a") + appender.append("b", "c") + appender.append("d") + await appender.flush() + assert store.batches == [4] + assert await _bodies(stream) == ["a", "b", "c", "d"] + + +async def test_flush_returns_the_last_offset(client: Client): + stream = await _create_stream(client, DictStreamStore()) + appender = BufferedAppender(stream) + assert await appender.flush() is None + appender.append("a", "b") + offset = await appender.flush() + assert offset is not None + assert offset == (await stream.describe()).last_offset + + +async def test_appends_during_a_write_go_in_the_next_batch(client: Client): + store = RecordingStore() + stream = await _create_stream(client, store) + store.release = asyncio.Event() + appender = BufferedAppender(stream, max_delay=_NEVER) + appender.append("a") + first = asyncio.create_task(appender.flush()) + await store.append_started.wait() + appender.append("b") + appender.append("c") + assert store.batches == [1] + store.release.set() + await first + await appender.flush() + assert store.batches == [1, 2] + assert await _bodies(stream) == ["a", "b", "c"] + + +async def test_a_large_buffer_goes_as_batches_of_at_most_1000(client: Client): + store = RecordingStore() + stream = await _create_stream(client, store) + appender = BufferedAppender(stream, max_delay=_NEVER) + values = [str(index) for index in range(2500)] + appender.append(*values) + # A full batch doesn't wait for the delay or a flush. + await store.append_started.wait() + await appender.flush() + assert store.batches == [1000, 1000, 500] + assert await _bodies(stream) == values + + +async def test_max_batch_records_cuts_the_batches(client: Client): + store = RecordingStore() + stream = await _create_stream(client, store) + appender = BufferedAppender(stream, max_delay=_NEVER, max_batch_records=2) + values = ["a", "b", "c", "d", "e"] + appender.append(*values) + await appender.flush() + assert store.batches == [2, 2, 1] + assert await _bodies(stream) == values + + +async def test_the_timer_alone_writes_the_batch(client: Client): + store = RecordingStore() + stream = await _create_stream(client, store) + appender = BufferedAppender(stream, max_delay=timedelta(milliseconds=10)) + appender.append("a", "b") + page = await stream.read_page(wait=timedelta(seconds=10)) + assert [record.body for record in page.records] == ["a", "b"] + assert store.batches == [2] + + +async def test_a_failed_batch_stops_the_appender(client: Client): + store = RecordingStore() + stream = await _create_stream(client, store) + await stream.close() + store.release = asyncio.Event() + appender = BufferedAppender(stream, max_delay=_NEVER) + appender.append("a") + first = asyncio.create_task(appender.flush()) + await store.append_started.wait() + appender.append("b", "c") + store.release.set() + err = await _stream_error(first) + assert err.reason == StreamErrorReason.STREAM_ERROR_REASON_CLOSED + assert await _stream_error(appender.flush()) is err + with pytest.raises(type(err)) as later: + appender.append("d") + assert later.value is err + assert appender.unwritten == 3 + assert store.batches == [1] + + +async def test_async_with_flushes_at_its_end(client: Client): + store = RecordingStore() + stream = await _create_stream(client, store) + async with BufferedAppender(stream, max_delay=_NEVER) as appender: + appender.append("a", "b") + assert store.batches == [2] + assert await _bodies(stream) == ["a", "b"] + + +async def test_async_with_keeps_the_error_of_its_block(client: Client): + store = RecordingStore() + stream = await _create_stream(client, store) + await stream.close() + appender = BufferedAppender(stream, max_delay=_NEVER) + with pytest.raises(RuntimeError, match="the block failed"): + async with appender: + appender.append("a") + raise RuntimeError("the block failed") + # The flush at the end of the block ran and failed. + assert store.batches == [1] + assert appender.unwritten == 1 + err = await _stream_error(appender.flush()) + assert err.reason == StreamErrorReason.STREAM_ERROR_REASON_CLOSED + + +async def test_append_without_a_value_fails(client: Client): + stream = _with_store(client, DictStreamStore()).get_stream( + StreamRef(id=f"stream-{uuid.uuid4()}"), body_type=str + ) + appender = BufferedAppender(stream) + with pytest.raises(TypeError, match="one or more values"): + appender.append() + with pytest.raises(ValueError, match="max_delay must not be negative"): + BufferedAppender(stream, max_delay=timedelta(seconds=-1)) + + +async def test_max_batch_records_must_be_from_1_to_1000(client: Client): + stream = _with_store(client, DictStreamStore()).get_stream( + StreamRef(id=f"stream-{uuid.uuid4()}"), body_type=str + ) + for value in (0, 1001): + with pytest.raises(ValueError) as err: + BufferedAppender(stream, max_batch_records=value) + assert ( + str(err.value) == f"max_batch_records must be from 1 to 1000, not {value}" + ) + for value in (1, 1000): + BufferedAppender(stream, max_batch_records=value) diff --git a/tests/contrib/streams/test_producer.py b/tests/contrib/streams/test_producer.py new file mode 100644 index 0000000000..86b6d922c9 --- /dev/null +++ b/tests/contrib/streams/test_producer.py @@ -0,0 +1,226 @@ +from __future__ import annotations + +import dataclasses +import uuid +from datetime import datetime, timedelta, timezone + +import pytest + +import temporalio.activity +import temporalio.api.stream.v1 +from temporalio import activity, workflow +from temporalio.client import Client +from temporalio.common import RetryPolicy +from temporalio.contrib.streams import as_producer, current_producer +from temporalio.converter import DataConverter +from temporalio.nexus.system.stream_service import ( + Producer, + ProducerKindClient, + ProducerKindLocalActivity, + ProducerKindStandaloneActivity, + ProducerKindWorkflow, + ProducerKindWorkflowActivity, + StreamRecord, + StreamRef, + WorkflowProducer, +) +from temporalio.nexus.system.stream_service.client import Stream +from temporalio.testing import ActivityEnvironment +from tests.helpers import new_worker +from tests.helpers.streams import DictStreamStore +from tests.nexus.test_stream_store import _create_stream, _with_store + +_SCHEDULED = datetime(2026, 11, 12, 10, 0, 0, 123456, tzinfo=timezone.utc) + +# One `activity.Info` for each kind of Activity that can write to a Stream. +_ACTIVITY_INFOS = { + "workflow_activity": dataclasses.replace( + ActivityEnvironment.default_info(), + activity_id="step", + attempt=2, + workflow_run_id="run-1", + scheduled_time=_SCHEDULED, + current_attempt_scheduled_time=_SCHEDULED + timedelta(seconds=30), + ), + "standalone_activity": dataclasses.replace( + ActivityEnvironment.default_info(), + activity_id="job", + activity_run_id="job-run", + attempt=2, + workflow_id=None, + workflow_namespace=None, + workflow_run_id=None, + workflow_type=None, + current_attempt_scheduled_time=_SCHEDULED, + ), + "local_activity": dataclasses.replace( + ActivityEnvironment.default_info(), + activity_id="3", + attempt=2, + is_local=True, + workflow_run_id="run-1", + workflow_task=temporalio.activity.WorkflowTaskAttempt( + started_event_id=7, attempt=1, started_time=_SCHEDULED + ), + ), +} + +_ACTIVITY_KINDS = { + "workflow_activity": ProducerKindWorkflowActivity, + "standalone_activity": ProducerKindStandaloneActivity, + "local_activity": ProducerKindLocalActivity, +} + + +def _activity_producer(kind: str) -> Producer: + env = ActivityEnvironment() + env.info = _ACTIVITY_INFOS[kind] + return env.run(current_producer) + + +def _producer_of_each_kind() -> dict[str, Producer]: + return { + "workflow": Producer( + id="wf:run-1", kind=ProducerKindWorkflow(WorkflowProducer(run_id="run-1")) + ), + **{kind: _activity_producer(kind) for kind in _ACTIVITY_INFOS}, + "client": current_producer(), + } + + +async def _records(stream: Stream[str]) -> list[StreamRecord[str]]: + return [record async for record in stream.read()] + + +async def test_current_producer_in_other_code_is_the_client_producer(client: Client): + stream = await _create_stream(client, DictStreamStore()) + await stream.append("a") + [record] = await _records(stream) + producer = current_producer() + assert record.producer == producer + assert isinstance(producer.kind, ProducerKindClient) + + +@pytest.mark.parametrize("kind", list(_ACTIVITY_INFOS)) +async def test_current_producer_in_an_activity_is_its_attempt_producer( + client: Client, kind: str +): + stream = await _create_stream(client, DictStreamStore()) + + async def append_and_give_producer() -> Producer: + await stream.append("a") + return current_producer() + + env = ActivityEnvironment() + env.info = _ACTIVITY_INFOS[kind] + producer = await env.run(append_and_give_producer) + [record] = await _records(stream) + assert record.producer == producer + assert isinstance(producer.kind, _ACTIVITY_KINDS[kind]) + + +@workflow.defn +class CurrentProducerWorkflow: + @workflow.run + async def run(self) -> str: + try: + current_producer() + except RuntimeError as err: + return str(err) + return "no error" + + +async def test_current_producer_in_workflow_code_raises(client: Client): + async with new_worker(client, CurrentProducerWorkflow) as worker: + message = await client.execute_workflow( + CurrentProducerWorkflow.run, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + assert message == ( + "Workflow code can't call a Stream in this version. Call it from an Activity " + "through activity.client()." + ) + + +@pytest.mark.parametrize("kind", ["workflow", *_ACTIVITY_INFOS, "client"]) +async def test_producer_round_trips_through_the_default_data_converter(kind: str): + producer = _producer_of_each_kind()[kind] + converter = DataConverter.default + [payload] = await converter.encode([producer]) + assert payload.metadata["messageType"] == b"temporal.api.stream.v1.Producer" + # A Worker decodes an Activity's arguments and its result with their type hints. + assert await converter.decode([payload], [Producer]) == [producer] + # It decodes heartbeat details with no type hint, so they give the proto message. + [detail] = await converter.decode([payload]) + assert isinstance(detail, temporalio.api.stream.v1.Producer) + assert as_producer(detail) == producer + + +def test_as_producer_keeps_a_producer_and_refuses_other_values(): + producer = current_producer() + assert as_producer(producer) is producer + with pytest.raises(TypeError, match="not str"): + as_producer("client:1") + + +@activity.defn +async def append_once_per_attempt(stream_id: str, keep_producer: bool) -> None: + info = activity.info() + stream = activity.client().get_stream(StreamRef(id=stream_id), body_type=str) + if info.attempt == 1: + await stream.append("first") + activity.heartbeat(current_producer()) + raise RuntimeError("Failed the first attempt") + producer = as_producer(info.heartbeat_details[0]) if keep_producer else None + await stream.append("second", producer=producer) + + +@workflow.defn +class AppendOncePerAttemptWorkflow: + @workflow.run + async def run(self, stream_id: str, keep_producer: bool) -> None: + await workflow.execute_activity( + append_once_per_attempt, + args=[stream_id, keep_producer], + start_to_close_timeout=timedelta(seconds=10), + heartbeat_timeout=timedelta(seconds=10), + retry_policy=RetryPolicy( + initial_interval=timedelta(milliseconds=10), maximum_attempts=2 + ), + ) + + +@pytest.mark.parametrize("keep_producer", [True, False]) +async def test_activity_retry_appends_as_its_stored_producer( + client: Client, keep_producer: bool +): + stream_client = _with_store(client, DictStreamStore()) + stream = await stream_client.create_stream( + id=f"stream-{uuid.uuid4()}", body_type=str + ) + async with new_worker( + stream_client, + AppendOncePerAttemptWorkflow, + activities=[append_once_per_attempt], + ) as worker: + # A FENCED append would fail the second and last attempt, and so the Workflow. + await stream_client.execute_workflow( + AppendOncePerAttemptWorkflow.run, + args=[stream.ref.id, keep_producer], + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + first, second = await _records(stream) + assert [first.body, second.body] == ["first", "second"] + assert first.producer is not None and second.producer is not None + assert first.producer.id == second.producer.id + assert isinstance(first.producer.kind, ProducerKindWorkflowActivity) + assert isinstance(second.producer.kind, ProducerKindWorkflowActivity) + first_epoch = first.producer.kind.value.epoch + second_epoch = second.producer.kind.value.epoch + if keep_producer: + # The retry continues the first attempt's epoch. + assert second_epoch == first_epoch + else: + assert (first_epoch.attempt, second_epoch.attempt) == (1, 2) diff --git a/tests/helpers/streams.py b/tests/helpers/streams.py new file mode 100644 index 0000000000..f2a3aad34e --- /dev/null +++ b/tests/helpers/streams.py @@ -0,0 +1,260 @@ +"""Stream Stores that the Stream tests share.""" + +from __future__ import annotations + +import asyncio +import contextlib +import dataclasses +from collections.abc import Awaitable, Callable +from typing import TypeVar + +from google.protobuf.message import Message +from google.protobuf.timestamp_pb2 import Timestamp + +from temporalio.api.enums.v1 import ( + StreamErrorReason, + StreamIdConflictPolicy, + StreamStatus, +) +from temporalio.api.stream.v1 import ( + Producer, + StreamRecord, + StreamRef, + WorkflowTaskEpoch, +) +from temporalio.api.streamservice.v1 import ( + AppendRecordsRequest, + AppendRecordsResponse, + CloseStreamRequest, + CloseStreamResponse, + CreateStreamRequest, + CreateStreamResponse, + DescribeStreamRequest, + DescribeStreamResponse, + ReadRecordsRequest, + ReadRecordsResponse, +) +from temporalio.client import StreamStore +from temporalio.exceptions import StreamError + +_RequestT = TypeVar("_RequestT", bound=Message) +_ResponseT = TypeVar("_ResponseT", bound=Message) + + +class RecordingStreamStore(StreamStore): + """A Store that passes each call to another Store and keeps the operation names. + + It has the name of the Store that it wraps, since that Store writes its own name + in the ref of a create. A test overrides :py:meth:`before_call` to fail a call or + to hold it. + """ + + def __init__(self, inner: StreamStore) -> None: + self.inner = inner + self.operations: list[str] = [] + + @property + def name(self) -> str: + return self.inner.name + + async def before_call(self, operation: str, request: Message) -> None: + del operation, request + + async def _forward( + self, + operation: str, + request: _RequestT, + call: Callable[[_RequestT], Awaitable[_ResponseT]], + ) -> _ResponseT: + self.operations.append(operation) + await self.before_call(operation, request) + return await call(request) + + async def create_stream(self, request: CreateStreamRequest) -> CreateStreamResponse: + return await self._forward("CreateStream", request, self.inner.create_stream) + + async def append_records( + self, request: AppendRecordsRequest + ) -> AppendRecordsResponse: + return await self._forward("AppendRecords", request, self.inner.append_records) + + async def read_records(self, request: ReadRecordsRequest) -> ReadRecordsResponse: + return await self._forward("ReadRecords", request, self.inner.read_records) + + async def close_stream(self, request: CloseStreamRequest) -> CloseStreamResponse: + return await self._forward("CloseStream", request, self.inner.close_stream) + + async def describe_stream( + self, request: DescribeStreamRequest + ) -> DescribeStreamResponse: + return await self._forward( + "DescribeStream", request, self.inner.describe_stream + ) + + +@dataclasses.dataclass +class _LastBatch: + epoch: tuple[int, ...] + sequence: int + response: AppendRecordsResponse + + +@dataclasses.dataclass +class _DictStream: + request_id: str + records: list[StreamRecord] = dataclasses.field(default_factory=list) + closed: bool = False + # The id of a Producer is unique in one Stream. + last_batches: dict[str, _LastBatch] = dataclasses.field(default_factory=dict) + + +def _offset(index: int) -> bytes: + return index.to_bytes(8, "big") + + +def _epoch(producer: Producer) -> tuple[int, ...]: + """Give a key that orders the epochs of one Producer, as ``Producer`` documents it.""" + kind = producer.WhichOneof("kind") + task = WorkflowTaskEpoch() + scheduled = Timestamp() + attempt = 0 + if kind == "workflow": + task = producer.workflow.epoch + elif kind == "local_activity": + task = producer.local_activity.epoch.task + attempt = producer.local_activity.epoch.attempt + elif kind in ("workflow_activity", "standalone_activity"): + scheduled = getattr(producer, kind).epoch.attempt_scheduled_time + attempt = getattr(producer, kind).epoch.attempt + return ( + task.started_event_id, + task.attempt, + task.started_time.seconds, + task.started_time.nanos, + scheduled.seconds, + scheduled.nanos, + attempt, + ) + + +class DictStreamStore(StreamStore): + """A Store that keeps its Streams in a dict, for the tests. + + It has no Retention and no expiry, and it checks no request. It resends the answer + of a batch that comes again with the same Producer, epoch and sequence, and it + fences a Producer whose epoch is lower than its newest one. A read waits for a new + Record up to the ``wait`` of its request. + """ + + def __init__(self, name: str = "dict") -> None: + self._name = name + self.streams: dict[str, _DictStream] = {} + self._changed = asyncio.Condition() + + @property + def name(self) -> str: + return self._name + + def _stream(self, ref: StreamRef) -> _DictStream: + stream = self.streams.get(ref.id) + if stream is None: + raise StreamError( + f"No Stream '{ref.id}'", + reason=StreamErrorReason.STREAM_ERROR_REASON_NOT_FOUND, + ) + return stream + + async def _notify(self) -> None: + async with self._changed: + self._changed.notify_all() + + async def create_stream(self, request: CreateStreamRequest) -> CreateStreamResponse: + ref = StreamRef(id=request.stream_ref.id, store=self.name) + stream = self.streams.get(ref.id) + if stream is None: + self.streams[ref.id] = _DictStream(request_id=request.request_id) + return CreateStreamResponse(ref=ref, created=True) + if stream.request_id == request.request_id: + return CreateStreamResponse(ref=ref, created=True) + if ( + request.id_conflict_policy + == StreamIdConflictPolicy.STREAM_ID_CONFLICT_POLICY_USE_EXISTING + ): + return CreateStreamResponse(ref=ref, created=False) + raise StreamError( + f"The Stream '{ref.id}' exists", + reason=StreamErrorReason.STREAM_ERROR_REASON_ALREADY_EXISTS, + ) + + async def append_records( + self, request: AppendRecordsRequest + ) -> AppendRecordsResponse: + stream = self._stream(request.ref) + epoch = _epoch(request.producer) + last = stream.last_batches.get(request.producer.id) + previous = last if last is not None and last.epoch == epoch else None + sequence = request.sequence or (previous.sequence + 1 if previous else 1) + if previous is not None and sequence == previous.sequence: + return previous.response + if stream.closed: + raise StreamError( + f"The Stream '{request.ref.id}' is closed", + reason=StreamErrorReason.STREAM_ERROR_REASON_CLOSED, + ) + if last is not None and epoch < last.epoch: + raise StreamError( + "The epoch is lower than the newest one of the Producer", + reason=StreamErrorReason.STREAM_ERROR_REASON_FENCED, + ) + first = len(stream.records) + 1 + for record in request.records: + stream.records.append( + StreamRecord( + producer=request.producer, + body=record.body, + offset=_offset(len(stream.records) + 1), + ) + ) + response = AppendRecordsResponse( + first_offset=_offset(first), last_offset=_offset(len(stream.records)) + ) + stream.last_batches[request.producer.id] = _LastBatch(epoch, sequence, response) + await self._notify() + return response + + async def read_records(self, request: ReadRecordsRequest) -> ReadRecordsResponse: + stream = self._stream(request.ref) + start = int.from_bytes(request.from_offset, "big") if request.from_offset else 0 + wait = request.wait.ToTimedelta() + if wait and start >= len(stream.records) and not stream.closed: + async with self._changed: + with contextlib.suppress(TimeoutError): + await asyncio.wait_for( + self._changed.wait_for( + lambda: len(stream.records) > start or stream.closed + ), + wait.total_seconds(), + ) + page = stream.records[start : start + (request.max_records or 1000)] + last_offset = page[-1].offset if page else request.from_offset + return ReadRecordsResponse( + records=page, + last_offset=last_offset, + done=stream.closed and start + len(page) == len(stream.records), + ) + + async def close_stream(self, request: CloseStreamRequest) -> CloseStreamResponse: + self._stream(request.ref).closed = True + await self._notify() + return CloseStreamResponse() + + async def describe_stream( + self, request: DescribeStreamRequest + ) -> DescribeStreamResponse: + stream = self._stream(request.ref) + return DescribeStreamResponse( + status=StreamStatus.STREAM_STATUS_CLOSED + if stream.closed + else StreamStatus.STREAM_STATUS_OPEN, + last_offset=stream.records[-1].offset if stream.records else b"", + ) diff --git a/tests/nexus/test_stream_store.py b/tests/nexus/test_stream_store.py new file mode 100644 index 0000000000..f9a582519f --- /dev/null +++ b/tests/nexus/test_stream_store.py @@ -0,0 +1,664 @@ +from __future__ import annotations + +import ast +import dataclasses +import subprocess +import sys +import uuid +from collections.abc import Awaitable, Callable +from datetime import datetime, timedelta, timezone +from pathlib import Path +from unittest import mock + +import google.protobuf.timestamp_pb2 +import pytest + +import temporalio +import temporalio.activity +import temporalio.api.stream.v1 +import temporalio.api.workflowservice.v1 +import temporalio.client._stream_store +import temporalio.common +import temporalio.converter._payload_converter +import temporalio.nexus.system +import temporalio.nexus.system._support +import temporalio.workflow +from temporalio.api.enums.v1 import StreamErrorReason +from temporalio.client import ( + Client, + StreamBatchLimits, + StreamConfig, + StreamSelectContext, + StreamStore, +) +from temporalio.client._stream_store import _STREAM_OPERATIONS, _request_ref +from temporalio.converter import DataConverter, StorageDriverActivityInfo +from temporalio.exceptions import StreamConfigError, StreamError +from temporalio.nexus.system.stream_service import ( + ClientProducer, + DescribeStreamRequest, + Producer, + ProducerKindClient, + ProducerKindLocalActivity, + ProducerKindStandaloneActivity, + ProducerKindWorkflowActivity, + StreamIdConflictPolicy, + StreamRef, + StreamService, + StreamStatus, + Timestamp, +) +from temporalio.nexus.system.stream_service.client import ( + Stream, + StreamServiceClientExtension, +) +from temporalio.testing import ActivityEnvironment, WorkflowEnvironment +from tests import DEV_SERVER_DOWNLOAD_VERSION +from tests.helpers.streams import DictStreamStore +from tests.test_converter import SimpleCodec + + +def _with_store(client: Client, store: StreamStore) -> Client: + return _with_config(client, StreamConfig(stores=[store])) + + +def _with_config(client: Client, stream_config: StreamConfig) -> Client: + config = client.config() + config["stream_config"] = stream_config + return Client(**config) + + +def _two_stores( + selector: Callable[[StreamSelectContext], StreamStore], +) -> tuple[DictStreamStore, DictStreamStore, StreamConfig]: + first = DictStreamStore(name="first") + second = DictStreamStore(name="second") + return first, second, StreamConfig(stores=[first, second], store_selector=selector) + + +async def _create_stream(client: Client, store: StreamStore) -> Stream[str]: + return await _with_store(client, store).create_stream( + id=f"stream-{uuid.uuid4()}", body_type=str + ) + + +async def _only_producer(stream: Stream[str]) -> Producer: + [record] = [record async for record in stream.read()] + assert record.producer is not None + return record.producer + + +async def _append_in_activity( + client: Client, info: temporalio.activity.Info +) -> Producer: + stream = await _create_stream(client, DictStreamStore()) + env = ActivityEnvironment() + env.info = info + await env.run(stream.append, "a") + return await _only_producer(stream) + + +def _timestamp(value: datetime) -> Timestamp: + proto = google.protobuf.timestamp_pb2.Timestamp() + proto.FromDatetime(value) + return Timestamp(seconds=proto.seconds, nanos=proto.nanos) + + +async def _stream_error(call: Awaitable[object]) -> StreamError: + with pytest.raises(StreamError) as err: + await call + return err.value + + +async def test_client_stream_uses_store(client: Client): + stream_client = _with_store(client, DictStreamStore()) + + stream = await stream_client.create_stream( + id=f"stream-{uuid.uuid4()}", body_type=str + ) + assert stream.created + assert stream.ref.store == "dict" + appended = await stream.append("a", "b") + assert appended.first_offset is not None + assert appended.last_offset is not None + assert appended.first_offset < appended.last_offset + records = [record async for record in stream.read()] + assert [(record.body, record.offset) for record in records] == [ + ("a", appended.first_offset), + ("b", appended.last_offset), + ] + + other = stream_client.get_stream(stream.ref, body_type=str) + assert [record.body async for record in other.read()] == ["a", "b"] + + described = await stream.describe() + assert described.status == StreamStatus.StreamStatusOpen + assert described.last_offset == appended.last_offset + + await stream.close() + described = await stream.describe() + assert described.status == StreamStatus.StreamStatusClosed + assert described.last_offset == appended.last_offset + page = await stream.read_page(from_offset=records[-1].offset) + assert page.done + assert page.records == [] + pages = [page async for page in stream.read_pages()] + assert pages[-1].done + assert [record.body for page in pages for record in page.records] == ["a", "b"] + + +async def test_client_stream_uses_a_store_written_from_scratch(client: Client): + store = DictStreamStore() + config = client.config() + config["data_converter"] = DataConverter(payload_codec=SimpleCodec()) + config["stream_config"] = StreamConfig(stores=[store]) + stream_client = Client(**config) + + stream_id = f"stream-{uuid.uuid4()}" + stream = await stream_client.create_stream(id=stream_id, body_type=str) + assert stream.created + assert stream.ref.store == "dict" + appended = await stream.append("a", "b") + assert [record.body async for record in stream.read()] == ["a", "b"] + # The Store gets payloads that the client already encoded with the codec. + stored = store.streams[stream_id].records + assert [record.body.metadata["simple-codec"] for record in stored] == [ + b"true", + b"true", + ] + described = await stream.describe() + assert described.status == StreamStatus.StreamStatusOpen + assert described.last_offset == appended.last_offset + err = await _stream_error(stream_client.create_stream(id=stream_id, body_type=str)) + assert err.reason == StreamErrorReason.STREAM_ERROR_REASON_ALREADY_EXISTS + + await stream.close() + err = await _stream_error(stream.append("c")) + assert err.reason == StreamErrorReason.STREAM_ERROR_REASON_CLOSED + + +def test_stream_operations_match_the_stream_service(): + service = temporalio.client._stream_store._STREAM_SERVICE + assert set(_STREAM_OPERATIONS) == set(service.operation_definitions) + for name, operation in _STREAM_OPERATIONS.items(): + definition = service.operation_definitions[name] + for model, message in ( + (definition.input_type, operation.request_type), + (definition.output_type, operation.response_type), + ): + converter = ( + temporalio.converter._payload_converter._get_transfer_type_converter( + model + ) + ) + assert converter is not None + assert converter.transfer_type is message + assert callable(getattr(StreamStore, operation.store_method)) + + +async def test_client_strict_create_of_existing_stream_fails(client: Client): + stream_client = _with_store(client, DictStreamStore()) + stream_id = f"stream-{uuid.uuid4()}" + await stream_client.create_stream(id=stream_id, body_type=str) + err = await _stream_error(stream_client.create_stream(id=stream_id, body_type=str)) + assert err.reason == StreamErrorReason.STREAM_ERROR_REASON_ALREADY_EXISTS + assert err.non_retryable + existing = await stream_client.create_stream( + id=stream_id, + body_type=str, + id_conflict_policy=StreamIdConflictPolicy.StreamIdConflictPolicyUseExisting, + ) + # The response's `created` is false, which the generated model reads as unset. + assert not existing.created + + +async def test_client_read_or_describe_of_unknown_stream_fails(client: Client): + stream_client = _with_store(client, DictStreamStore()) + stream = stream_client.get_stream( + StreamRef(id=f"stream-{uuid.uuid4()}"), body_type=str + ) + for call in (stream.read_page(), stream.describe()): + err = await _stream_error(call) + assert err.reason == StreamErrorReason.STREAM_ERROR_REASON_NOT_FOUND + + +async def test_client_append_after_close_fails(client: Client): + stream = await _create_stream(client, DictStreamStore()) + await stream.append("a") + await stream.close() + err = await _stream_error(stream.append("b")) + assert err.reason == StreamErrorReason.STREAM_ERROR_REASON_CLOSED + assert err.type == "StreamError" + + +async def test_client_stream_store_uses_payload_codec(client: Client): + store = DictStreamStore() + config = client.config() + config["data_converter"] = DataConverter(payload_codec=SimpleCodec()) + config["stream_config"] = StreamConfig(stores=[store]) + stream_client = Client(**config) + + stream = await stream_client.create_stream( + id=f"stream-{uuid.uuid4()}", body_type=str + ) + await stream.append("a", "b") + # A client without the codec reads the bodies as the Store keeps them. + raw = _with_store(client, store).get_stream( + StreamRef(id=stream.ref.id), body_type=temporalio.common.RawValue + ) + stored = [record.body.payload async for record in raw.read()] + assert [body.metadata["simple-codec"] for body in stored] == [b"true", b"true"] + assert [record.body async for record in stream.read()] == ["a", "b"] + + +async def test_client_append_has_client_producer(client: Client): + stream = await _create_stream(client, DictStreamStore()) + await stream.append("a") + producer = await _only_producer(stream) + assert producer.id.startswith("client:") + assert isinstance(producer.kind, ProducerKindClient) + + +async def test_workflow_activity_append_has_activity_producer(client: Client): + scheduled = datetime(2026, 11, 12, 10, 0, 0, 123456, tzinfo=timezone.utc) + attempt_scheduled = scheduled + timedelta(seconds=30) + producer = await _append_in_activity( + client, + dataclasses.replace( + ActivityEnvironment.default_info(), + activity_id="step:1", + attempt=3, + workflow_run_id="run-1", + scheduled_time=scheduled, + current_attempt_scheduled_time=attempt_scheduled, + ), + ) + epoch_nanos = ( + (scheduled - datetime(1970, 1, 1, tzinfo=timezone.utc)) + // timedelta(microseconds=1) + * 1000 + ) + assert producer.id == f"act:run-1:step%3A1:{epoch_nanos}" + assert isinstance(producer.kind, ProducerKindWorkflowActivity) + assert producer.kind.value.run_id == "run-1" + assert producer.kind.value.epoch.attempt == 3 + assert producer.kind.value.epoch.attempt_scheduled_time == _timestamp( + attempt_scheduled + ) + + +async def test_standalone_activity_append_has_activity_producer(client: Client): + attempt_scheduled = datetime(2026, 11, 12, 10, 0, 30, tzinfo=timezone.utc) + producer = await _append_in_activity( + client, + dataclasses.replace( + ActivityEnvironment.default_info(), + activity_id="job", + activity_run_id="job-run", + attempt=2, + workflow_id=None, + workflow_namespace=None, + workflow_run_id=None, + workflow_type=None, + current_attempt_scheduled_time=attempt_scheduled, + ), + ) + assert producer.id == "sa:job:job-run" + assert isinstance(producer.kind, ProducerKindStandaloneActivity) + assert not producer.kind.value.run_id + assert producer.kind.value.epoch.attempt == 2 + assert producer.kind.value.epoch.attempt_scheduled_time == _timestamp( + attempt_scheduled + ) + + +async def test_local_activity_append_has_local_activity_producer(client: Client): + task_started = datetime(2026, 11, 12, 10, 0, 0, 123456, tzinfo=timezone.utc) + producer = await _append_in_activity( + client, + dataclasses.replace( + ActivityEnvironment.default_info(), + activity_id="3", + attempt=2, + is_local=True, + workflow_run_id="run:1", + workflow_task=temporalio.activity.WorkflowTaskAttempt( + started_event_id=7, attempt=1, started_time=task_started + ), + ), + ) + assert producer.id == "la:run%3A1:3" + assert isinstance(producer.kind, ProducerKindLocalActivity) + assert producer.kind.value.run_id == "run:1" + assert producer.kind.value.epoch.task.started_event_id == 7 + assert producer.kind.value.epoch.task.attempt == 1 + assert producer.kind.value.epoch.task.started_time == _timestamp(task_started) + assert producer.kind.value.epoch.attempt == 2 + + +async def test_local_activity_append_without_workflow_task_fails(client: Client): + with pytest.raises(RuntimeError, match="Workflow Task"): + await _append_in_activity( + client, + dataclasses.replace(ActivityEnvironment.default_info(), is_local=True), + ) + + +async def test_explicit_producer_is_sent(client: Client): + stream = await _create_stream(client, DictStreamStore()) + with mock.patch.object( + temporalio.nexus.system._support, "default_producer" + ) as default_producer: + await stream.append( + "a", producer=Producer(id="mine", kind=ProducerKindClient(ClientProducer())) + ) + default_producer.assert_not_called() + producer = await _only_producer(stream) + assert producer.id == "mine" + assert isinstance(producer.kind, ProducerKindClient) + + +async def test_client_append_with_sequence_writes_once(client: Client): + stream = await _create_stream(client, DictStreamStore()) + producer = Producer(id="mine", kind=ProducerKindClient(ClientProducer())) + first = await stream.append("a", "b", producer=producer, sequence=5) + second = await stream.append("a", "b", producer=producer, sequence=5) + assert second == first + records = [record async for record in stream.read()] + assert [record.body for record in records] == ["a", "b"] + assert [record.offset for record in records] == [ + first.first_offset, + first.last_offset, + ] + + +def test_client_stream_methods_are_generated(): + assert "create_stream" not in vars(Client) + assert "get_stream" not in vars(Client) + assert Client.create_stream is StreamServiceClientExtension.create_stream + assert Client.get_stream is StreamServiceClientExtension.get_stream + + +def test_no_hand_written_stream_classes(): + root = Path(temporalio.__file__).parent + generated = root / "nexus" / "system" / "stream_service" + names = {"Stream", "WorkflowStream", "StreamCalls", "StreamClient"} + found: list[tuple[str, str]] = [] + for source in sorted(root.rglob("*.py")): + if generated in source.parents: + continue + for node in ast.walk(ast.parse(source.read_text(encoding="utf8"))): + if isinstance(node, ast.ClassDef) and node.name in names: + found.append((source.relative_to(root).as_posix(), node.name)) + # The Workflow Streams contrib library is older than the generated Stream API and + # doesn't use it. + found.remove(("contrib/workflow_streams/_stream.py", "WorkflowStream")) + assert found == [] + for name in ("Stream", "WorkflowStream"): + value = getattr(temporalio.workflow, name, None) + assert value is None or value.__module__.startswith( + "temporalio.nexus.system.stream_service." + ) + + +def test_each_stream_module_can_be_the_first_import(): + modules = [ + "temporalio.nexus.system.stream_service.workflow", + "temporalio.nexus.system.stream_service.client", + "temporalio.nexus.system.stream_service", + "temporalio.nexus.system", + "temporalio.workflow", + "temporalio.client", + "temporalio.worker", + ] + failures: dict[str, str] = {} + for module in modules: + # Only a new interpreter has no module loaded, so the import order is real. + result = subprocess.run( + [sys.executable, "-c", f"import {module}"], + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + failures[module] = result.stderr.strip().splitlines()[-1] + assert failures == {} + + +async def test_client_without_stream_store_raises_and_sends_nothing(client: Client): + requests: list[ + temporalio.api.workflowservice.v1.StartNexusOperationExecutionRequest + ] = [] + + async def start_nexus_operation_execution( + req: temporalio.api.workflowservice.v1.StartNexusOperationExecutionRequest, + **_: object, + ) -> temporalio.api.workflowservice.v1.StartNexusOperationExecutionResponse: + requests.append(req) + return temporalio.api.workflowservice.v1.StartNexusOperationExecutionResponse( + run_id="run" + ) + + nexus_client = client.create_nexus_client( + StreamService, temporalio.nexus.system.TEMPORAL_SYSTEM_ENDPOINT + ) + with mock.patch.object( + client.workflow_service, + "start_nexus_operation_execution", + start_nexus_operation_execution, + ): + with pytest.raises(StreamConfigError) as err: + await nexus_client.start_operation( + StreamService.describe_stream, + DescribeStreamRequest(ref=StreamRef(id="stream")), + id="operation", + ) + # The generated client method goes the same way. + with pytest.raises(StreamConfigError): + await client.create_stream(id="stream", body_type=str) + assert str(err.value) == ( + "This client has no Stream Store. Give one in " + "stream_config=StreamConfig(stores=[...])." + ) + assert requests == [] + + +def test_stream_config_refuses_a_bad_set_of_stores(): + with pytest.raises(StreamConfigError, match="at least one Store"): + StreamConfig(stores=[]) + same_name = [DictStreamStore(name="a"), DictStreamStore(name="a")] + with pytest.raises(StreamConfigError, match="more than one Store named 'a'"): + StreamConfig(stores=same_name, store_selector=lambda _: same_name[0]) + assert not issubclass(StreamConfigError, StreamError) + + +def test_stream_config_takes_several_stores_with_no_selector(): + first, second = DictStreamStore(name="a"), DictStreamStore(name="b") + stream_config = StreamConfig(stores=[first, second]) + assert stream_config.store_selector is None + assert stream_config._store_for(StreamRef(id="stream", store="b"), None) is second + with pytest.raises(StreamConfigError, match="more than one Store"): + stream_config._store_for(StreamRef(id="stream"), None) + + +_BATCH_LIMIT_DEFAULTS = { + "max_record_bytes": 1536 * 1024, + "max_batch_bytes": 4 * 1024 * 1024, + "max_batch_records": 1000, +} + + +def test_stream_batch_limits_default_to_the_largest_batch(): + assert dataclasses.asdict(StreamBatchLimits()) == _BATCH_LIMIT_DEFAULTS + + +@pytest.mark.parametrize("field,default", _BATCH_LIMIT_DEFAULTS.items()) +def test_stream_batch_limits_take_a_value_from_1_to_the_default( + field: str, default: int +): + for value in (1, default): + assert getattr(StreamBatchLimits(**{field: value}), field) == value + for value in (0, default + 1): + with pytest.raises(ValueError) as err: + StreamBatchLimits(**{field: value}) + assert str(err.value) == f"{field} must be from 1 to {default}, not {value}" + + +def test_stream_store_without_its_own_limits_gives_the_default(): + assert DictStreamStore().limits == StreamBatchLimits() + + +async def test_client_create_with_store_on_several_stores_needs_no_selector( + client: Client, +): + first = DictStreamStore(name="first") + second = DictStreamStore(name="second") + stream_client = _with_config(client, StreamConfig(stores=[first, second])) + stream = await stream_client.create_stream( + id=f"stream-{uuid.uuid4()}", store="second", body_type=str + ) + assert stream.ref.store == "second" + await stream.append("a") + in_second = _with_store(client, second).get_stream(stream.ref, body_type=str) + assert [record.body async for record in in_second.read()] == ["a"] + in_first = _with_store(client, first).get_stream( + StreamRef(id=stream.ref.id), body_type=str + ) + err = await _stream_error(in_first.describe()) + assert err.reason == StreamErrorReason.STREAM_ERROR_REASON_NOT_FOUND + + +async def test_client_create_without_store_on_several_stores_fails(client: Client): + stream_config = StreamConfig( + stores=[DictStreamStore(name="first"), DictStreamStore(name="second")] + ) + stream_client = _with_config(client, stream_config) + with pytest.raises(StreamConfigError) as err: + await stream_client.create_stream(id="stream", body_type=str) + assert str(err.value) == ( + "StreamConfig has more than one Store and no store_selector, and the call " + "on the Stream 'stream' names no Store. Name one with store=, or set " + "StreamConfig.store_selector." + ) + + +def test_client_refuses_a_store_in_plugins(client: Client): + config = client.config() + config["plugins"] = [DictStreamStore()] # type: ignore[list-item] + with pytest.raises(StreamConfigError, match="plugins"): + Client(**config) + + +async def test_selector_picks_the_store_of_a_new_stream(client: Client): + contexts: list[StreamSelectContext] = [] + + def select(context: StreamSelectContext) -> StreamStore: + contexts.append(context) + return second + + first, second, stream_config = _two_stores(select) + stream_client = _with_config(client, stream_config) + stream_id = f"stream-{uuid.uuid4()}" + stream = await stream_client.create_stream(id=stream_id, body_type=str) + assert stream.ref.store == "second" + assert contexts == [StreamSelectContext(id=stream_id, target=None)] + + await stream_client.get_stream(stream.ref, body_type=str).append("a") + # A call through a ref that names its Store doesn't ask the selector. + assert len(contexts) == 1 + in_second = _with_store(client, second).get_stream(stream.ref, body_type=str) + assert [record.body async for record in in_second.read()] == ["a"] + in_first = _with_store(client, first).get_stream( + StreamRef(id=stream_id), body_type=str + ) + err = await _stream_error(in_first.describe()) + assert err.reason == StreamErrorReason.STREAM_ERROR_REASON_NOT_FOUND + + +async def test_selector_gets_the_activity_that_makes_the_call(client: Client): + contexts: list[StreamSelectContext] = [] + + def select(context: StreamSelectContext) -> StreamStore: + contexts.append(context) + return first + + first, _, stream_config = _two_stores(select) + stream_client = _with_config(client, stream_config) + stream_id = f"stream-{uuid.uuid4()}" + env = ActivityEnvironment() + env.info = dataclasses.replace( + ActivityEnvironment.default_info(), + activity_id="job", + activity_type="make_stream", + activity_run_id="job-run", + namespace="streams", + ) + + async def create() -> Stream[str]: + return await stream_client.create_stream(id=stream_id, body_type=str) + + stream = await env.run(create) + assert stream.ref.store == "first" + assert contexts == [ + StreamSelectContext( + id=stream_id, + target=StorageDriverActivityInfo( + id="job", type="make_stream", run_id="job-run", namespace="streams" + ), + ) + ] + + +async def test_selector_that_returns_another_store_fails(client: Client): + _, _, stream_config = _two_stores(lambda _: DictStreamStore(name="first")) + stream_client = _with_config(client, stream_config) + with pytest.raises(StreamConfigError, match="isn't one of"): + await stream_client.create_stream(id=f"stream-{uuid.uuid4()}", body_type=str) + + +async def test_ref_of_a_store_that_the_client_lacks_fails(client: Client): + stream_client = _with_store(client, DictStreamStore(name="first")) + stream = stream_client.get_stream( + StreamRef(id=f"stream-{uuid.uuid4()}", store="other"), body_type=str + ) + with pytest.raises(StreamConfigError, match="'other'"): + await stream.describe() + + +def test_request_without_a_stream_ref_fails(): + request = Producer(id="mine", kind=ProducerKindClient(ClientProducer())) + with pytest.raises(TypeError, match="Producer has no StreamRef"): + _request_ref(request) + + +async def test_start_local_gives_its_client_the_stream_config(): + stream_config = StreamConfig(stores=[DictStreamStore()]) + async with await WorkflowEnvironment.start_local( + stream_config=stream_config, + dev_server_download_version=DEV_SERVER_DOWNLOAD_VERSION, + ) as env: + assert env.client.config()["stream_config"] is stream_config + stream = await env.client.create_stream( + id=f"stream-{uuid.uuid4()}", body_type=str + ) + assert stream.ref.store == "dict" + + +def test_stream_error_carries_its_reason(): + err = StreamError( + "the Stream is closed", reason=StreamErrorReason.STREAM_ERROR_REASON_CLOSED + ) + assert err.type == "StreamError" + assert err.message == "the Stream is closed" + assert err.non_retryable + assert list(err.details) == [ + temporalio.api.stream.v1.StreamErrorDetails( + reason=StreamErrorReason.STREAM_ERROR_REASON_CLOSED + ) + ] + assert not StreamError( + "down", reason=StreamErrorReason.STREAM_ERROR_REASON_STORAGE_UNAVAILABLE + ).non_retryable + assert StreamError("x").reason == ( + StreamErrorReason.STREAM_ERROR_REASON_UNSPECIFIED + ) diff --git a/tests/nexus/test_stream_store_workflow.py b/tests/nexus/test_stream_store_workflow.py new file mode 100644 index 0000000000..c1df44daaa --- /dev/null +++ b/tests/nexus/test_stream_store_workflow.py @@ -0,0 +1,192 @@ +from __future__ import annotations + +import uuid +from datetime import timedelta + +import pytest + +from temporalio import activity, workflow +from temporalio.client import Client +from temporalio.nexus.system import TEMPORAL_SYSTEM_ENDPOINT +from temporalio.nexus.system.stream_service import ( + Producer, + ProducerKindLocalActivity, + ReadRecordsRequest, + StreamRecord, + StreamRef, + StreamService, +) +from temporalio.nexus.system.stream_service.workflow import create_stream, get_stream +from tests.helpers import assert_task_fail_eventually, new_worker +from tests.helpers.streams import DictStreamStore, RecordingStreamStore +from tests.nexus.test_stream_store import _with_store + +_BODIES = ["first-record-body", "second-record-body"] + + +@workflow.defn +class StreamCallWorkflow: + @workflow.run + async def run(self, operation: str) -> None: + stream = get_stream(StreamRef(id="stream"), body_type=str) + if operation == "CreateStream": + await create_stream(id="stream", body_type=str) + elif operation == "AppendRecords": + await stream.append("a") + elif operation == "CloseStream": + await stream.close() + elif operation == "DescribeStream": + await stream.describe() + else: + # The Workflow `Stream` has no `read`, but a Nexus client can start one. + nexus_client = workflow.create_nexus_client( + service=StreamService, endpoint=TEMPORAL_SYSTEM_ENDPOINT + ) + await nexus_client.start_operation( + operation, ReadRecordsRequest(ref=StreamRef[str](id="stream")) + ) + + +@pytest.mark.parametrize( + "operation", + ["CreateStream", "AppendRecords", "CloseStream", "DescribeStream", "ReadRecords"], +) +async def test_workflow_stream_call_fails_the_workflow_task( + client: Client, operation: str +): + store = RecordingStreamStore(DictStreamStore()) + stream_client = _with_store(client, store) + async with new_worker(stream_client, StreamCallWorkflow) as worker: + handle = await stream_client.start_workflow( + StreamCallWorkflow.run, + operation, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + execution_timeout=timedelta(seconds=30), + ) + await assert_task_fail_eventually( + handle, + message_contains=( + "Workflow code can't call a Stream in this version. Call it from an " + "Activity through activity.client()." + ), + ) + await handle.terminate() + # The Store of the Worker's client gets no call. + assert store.operations == [] + + +@activity.defn +async def append_values(ref: StreamRef[str], values: list[str]) -> None: + await activity.client().get_stream(ref, body_type=str).append(*values) + + +_fail_next_task = True + + +@workflow.defn(sandboxed=False) +class AppendThenFailTaskOnceWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str]) -> None: + await workflow.execute_local_activity( + append_values, + args=[ref, _BODIES], + start_to_close_timeout=timedelta(seconds=10), + ) + global _fail_next_task + if _fail_next_task: + _fail_next_task = False + raise RuntimeError("Failed the Workflow Task once") + + +@activity.defn +async def append_from_local_activity(stream_id: str) -> None: + stream = activity.client().get_stream(StreamRef(id=stream_id), body_type=str) + await stream.append("a") + + +@workflow.defn +class UserLocalActivityAppendWorkflow: + @workflow.run + async def run(self, stream_id: str) -> None: + await workflow.execute_local_activity( + append_from_local_activity, + stream_id, + start_to_close_timeout=timedelta(seconds=10), + ) + + +async def _read_all(client: Client, stream_id: str) -> list[StreamRecord[str]]: + stream = client.get_stream(StreamRef(id=stream_id), body_type=str) + return [record async for record in stream.read()] + + +def _local_activity_epoch(producer: Producer) -> tuple[int, int, int]: + # The Workflow Task, then the attempt of the Local Activity. + assert isinstance(producer.kind, ProducerKindLocalActivity) + local = producer.kind.value.epoch + return (local.task.started_event_id, local.task.attempt, local.attempt) + + +def _producers(records: list[StreamRecord[str]]) -> list[Producer]: + producers = [record.producer for record in records] + assert all(producer is not None for producer in producers) + return [producer for producer in producers if producer is not None] + + +async def test_local_activity_append_after_a_failed_workflow_task_has_a_higher_epoch( + client: Client, +): + global _fail_next_task + _fail_next_task = True + stream_client = _with_store(client, DictStreamStore()) + stream = await stream_client.create_stream( + id=f"stream-{uuid.uuid4()}", body_type=str + ) + async with new_worker( + stream_client, AppendThenFailTaskOnceWorkflow, activities=[append_values] + ) as worker: + await stream_client.execute_workflow( + AppendThenFailTaskOnceWorkflow.run, + stream.ref, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + records = await _read_all(stream_client, stream.ref.id) + # The failed Workflow Task records no marker, so its retry runs the Local Activity + # again and writes a second copy. + assert [record.body for record in records] == [*_BODIES, *_BODIES] + producers = _producers(records) + assert len({producer.id for producer in producers}) == 1 + first, second, third, fourth = ( + _local_activity_epoch(producer) for producer in producers + ) + assert first == second + assert third == fourth + assert third > first + + +async def test_user_local_activity_append_has_local_activity_producer(client: Client): + stream_client = _with_store(client, DictStreamStore()) + stream = await stream_client.create_stream( + id=f"stream-{uuid.uuid4()}", body_type=str + ) + async with new_worker( + stream_client, + UserLocalActivityAppendWorkflow, + activities=[append_from_local_activity], + ) as worker: + handle = await stream_client.start_workflow( + UserLocalActivityAppendWorkflow.run, + stream.ref.id, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + await handle.result() + [producer] = _producers(await _read_all(stream_client, stream.ref.id)) + run_id = handle.result_run_id + assert producer.id == f"la:{run_id}:1" + assert isinstance(producer.kind, ProducerKindLocalActivity) + assert producer.kind.value.run_id == run_id + assert producer.kind.value.epoch.task.started_event_id > 0 + assert producer.kind.value.epoch.attempt == 1