diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 12e3b8e566..da1b43d780 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -226,8 +226,9 @@ jobs: - run: uv tool install poethepoet - run: uv sync --all-extras - run: cargo install --locked nexgen --version 0.2.8 --features advanced --force - # The Stream package needs the read, read pages and read page methods of a poll, which only - # a nexgen fork has. + # The Stream package needs `@nexus.helper` with `sync=true`, which only a nexgen fork has. + # This commit of the fork also has the poll iterators of the Workflow target and the read, + # read pages and read page methods of a poll. - run: cargo install --locked --git https://github.com/moetemp/nexgen --rev 07b565b6d4339dc8a3f67ce2e9e72ba4925d7f9c --features advanced --root "$RUNNER_TEMP/nexgen-stream" nexgen - run: echo "NEX_GEN_STREAM_BIN=$RUNNER_TEMP/nexgen-stream/bin/nexgen" >> "$GITHUB_ENV" - run: poe build-develop diff --git a/changelog/added/chattering-ledger-otter.md b/changelog/added/chattering-ledger-otter.md index 1c8c58885f..a165dffa36 100644 --- a/changelog/added/chattering-ledger-otter.md +++ b/changelog/added/chattering-ledger-otter.md @@ -1,3 +1,3 @@ -Add the experimental Streams API. Client and Activity code create a Stream, append Records to it, read it and close it. Workflow code can't call a Stream in this version. A client takes its Stream Stores in `stream_config=StreamConfig(stores=[...])`, such as `RedisStreamStore` of the `temporalio-streams-redis` package, or `MemoryStreamStore` of the `temporalio-streams-memory` package in tests. +Add the experimental Streams API. Client and Activity code create a Stream, append Records to it, read it and close it. A client takes its Stream Stores in `stream_config=StreamConfig(stores=[...])`, such as `RedisStreamStore` of the `temporalio-streams-redis` package, or `MemoryStreamStore` of the `temporalio-streams-memory` package in tests. Add `temporalio.contrib.streams.current_producer` and `as_producer`. An Activity can store its Producer in its heartbeat details and pass it to the appends of its retry. The retry's Records then continue the first attempt's epoch, so a reader does not take them as a replacement of the first attempt's Records. A Workflow outbound interceptor sees a Stream operation only in its operation-specific method, such as `start_append_records`, and not in `start_system_nexus_operation`. An unknown Stream operation raises `ValueError`. diff --git a/changelog/added/patient-river-heron.md b/changelog/added/patient-river-heron.md new file mode 100644 index 0000000000..81f5177f33 --- /dev/null +++ b/changelog/added/patient-river-heron.md @@ -0,0 +1,11 @@ +Workflow code creates a Stream, appends Records to it, reads it, describes it and closes it, with `workflow.create_stream` and `workflow.get_stream`. A create and a describe run in the Store's Local Activity, with the options of `StreamConfig.workflow_call_options`. Core makes a read, an append and a close itself, with no Activity. It calls the Store as a task on the Worker's event loop, never on a Workflow thread. For Workflow code, a Store waits up to a read's `wait`, ends that wait on a cancel, gives offsets of 1 to 1024 bytes that compare in order, writes a resent batch once, and raises `StreamError` with a reason. A Store can refuse a lower epoch of a Producer with `STREAM_ERROR_REASON_FENCED`, and it doesn't have to. Core tries a call again for up to 30 s only on `STREAM_ERROR_REASON_STORAGE_UNAVAILABLE`. Any other exception reaches Workflow code on the first try, as a non-retryable `STREAM_ERROR_REASON_STORAGE_CORRUPT` with its type in the message. `Replayer` takes the `stream_config` of the Worker's client. +The Workflow `Stream` has `read`, `read_pages` and `read_page`, so Workflow code reads a Stream with the same calls as a client. Core reads the Store inside the Workflow Task. A page that Workflow code gets in one activation has at most 256 Records, and the rest come in the next activation with no wait. A loop that asks for its next page in the same activation keeps one read open, and a read that no code waits on closes before the activation completes, so it never holds a Workflow Task open. With a `wait`, a quiet Stream gives empty pages, so a loop on a Stream that can be quiet for long uses `read_pages` and leaves after an empty page. A call that waits on a quiet Stream holds its Workflow Task open, with one Workflow Task about every 8 s. The calls of one Workflow Task write one marker in History. `StreamConfig.workflow_read_replay_from` decides whether the marker holds the Records (`StreamReplaySource.HISTORY`) or only their offsets and a digest, which a replay reads from the Store again (`StreamReplaySource.STORE`, the default). With `HISTORY`, one Workflow Task's marker holds at most 1 MiB of Records, and the read goes on in the next task. +A Workflow's append runs in Core inside the Workflow Task, with no Activity. It writes as the Workflow, with its Workflow Task as the epoch. A `producer=` of another kind raises `StreamError` with the reason `STREAM_ERROR_REASON_INVALID`. +A Workflow's close runs in Core, with no Activity, and spans two Workflow Tasks. The task that calls `close` records the intent, Core closes the Stream after the server took that task, and the next task gives the result, so a failed task closes nothing. Await the close. A graceful shutdown of the Worker waits at most 10 s for a close that the code doesn't await, and a Worker that dies when the Workflow ends loses it. An append and a close of Workflow code always carry the Workflow Task as their epoch. An append is at-least-once. After a Worker crash or an eviction, the rerun of its task can write a second copy with the epoch of the new task. +Core records metrics for the Direct calls of a Stream, such as `temporal_stream_read_total`, `temporal_stream_read_records` and `temporal_stream_close_latency`, each with the attribute `stream_store`, and `temporal_stream_close_abandoned` for a close that a graceful shutdown gave up. +Add `StreamConfig.workflow_read_ahead_bytes`, the most bytes of one read's Records that wait in the Worker for Workflow code, 4 MiB by default. +Add `StreamConfig.workflow_read_page_records`, the most Records of one read that Workflow code gets in one activation. It is 256 by default and must be 1 to 1000. +Add `StreamConfig.workflow_read_min_gap`, the least time between two batches of one read. It has no gap by default and must be at most 1 s. A gap trades latency for fewer activations of Workflow code. +Add `temporalio.exceptions.StreamReplayError`. With it in `workflow_failure_exception_types`, a replay that can't read the Records of a read from the Store again fails the Workflow, and not only the Workflow Task. +Core tries a read, an append or a close of Workflow code again while its Store is unavailable, from 100 ms up to 2 s apart, for at most 30 s. Only then does Workflow code get the retryable `StreamError`. A replay tries its Store for 5 s. If the Store stays down, only the Workflow Task fails, with `StreamReplayUnavailable`, also with `StreamReplayError` in `workflow_failure_exception_types`. +In Workflow code, `temporalio.contrib.streams.current_producer` gives the Workflow's Producer. `BufferedAppender` is for client and Activity code, and it raises `RuntimeError` in Workflow code. diff --git a/scripts/gen_nexus_system_api.py b/scripts/gen_nexus_system_api.py index fdf38fa90c..224330d63a 100644 --- a/scripts/gen_nexus_system_api.py +++ b/scripts/gen_nexus_system_api.py @@ -28,9 +28,10 @@ workflow_init_path = base_dir / "temporalio" / "workflow" / "__init__.py" NEX_GEN_VERSION = "0.2.8" # The Stream package needs a newer nexgen, which rejects the other WITs. One pin is enough -# again once the api WITs and a nexgen release agree. The read, read pages and read page methods -# of a poll are only in commit 07b565b6 of the fork https://github.com/moetemp/nexgen. So -# NEX_GEN_STREAM_BIN points at a build of that commit. +# again once the api WITs and a nexgen release agree. The Stream WIT uses `@nexus.helper` with +# `sync=true` and `@nexus.support-type`, which only the fork https://github.com/moetemp/nexgen +# has. Its commit 07b565b6 also has the poll iterators of the Workflow target and the read, read +# pages and read page methods of a poll. So NEX_GEN_STREAM_BIN points at a build of that commit. NEX_GEN_STREAM_VERSION = "0.2.12" proto_files = [ gen_protos.api_proto_dir diff --git a/temporalio/api/stream/v1/message_pb2.pyi b/temporalio/api/stream/v1/message_pb2.pyi index 1d01036026..1e42e74a20 100644 --- a/temporalio/api/stream/v1/message_pb2.pyi +++ b/temporalio/api/stream/v1/message_pb2.pyi @@ -167,8 +167,8 @@ global___ActivityEpoch = ActivityEpoch @typing.final class WorkflowProducer(google.protobuf.message.Message): """Experimental. Might change in the future. - A Workflow run Producer. On the server's Streams, a Workflow writes as this Producer in its - Workflow Task. On a Store, Workflow code writes through a Local Activity, so its writes carry a + A Workflow run Producer. Workflow code that writes in its Workflow Task writes as this + Producer, with that task as the epoch. A write that it makes through a Local Activity has a Local Activity Producer. """ @@ -180,7 +180,9 @@ class WorkflowProducer(google.protobuf.message.Message): """Required. The run id of the Workflow run, not of its chain. Each run is its own Producer.""" @property def epoch(self) -> global___WorkflowTaskEpoch: - """The Workflow Task of the write.""" + """The Workflow Task of the write. Workflow code leaves it empty, and the SDK sets it before + the write goes to the Store. + """ def __init__( self, diff --git a/temporalio/bridge/Cargo.lock b/temporalio/bridge/Cargo.lock index 3285c012ca..bae80dc51c 100644 --- a/temporalio/bridge/Cargo.lock +++ b/temporalio/bridge/Cargo.lock @@ -181,6 +181,15 @@ version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bon" version = "3.10.3" @@ -265,7 +274,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.3.1", "rand_core 0.10.1", ] @@ -343,6 +352,15 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "cpufeatures" version = "0.3.1" @@ -376,6 +394,16 @@ version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "darling" version = "0.24.1" @@ -433,6 +461,16 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "dirs" version = "7.0.0" @@ -728,6 +766,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "gethostname" version = "1.1.0" @@ -2390,6 +2438,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -2593,6 +2652,7 @@ dependencies = [ "temporalio-client", "temporalio-common", "temporalio-sdk-core", + "temporalio-streams", "tokio", "tokio-rustls", "tokio-stream", @@ -2765,6 +2825,7 @@ dependencies = [ "temporalio-client", "temporalio-common", "temporalio-macros", + "temporalio-streams", "thiserror", "tokio", "tokio-stream", @@ -2776,6 +2837,19 @@ dependencies = [ "zip", ] +[[package]] +name = "temporalio-streams" +version = "0.1.0" +dependencies = [ + "async-trait", + "prost", + "prost-wkt-types", + "sha2", + "temporalio-common", + "thiserror", + "tonic", +] + [[package]] name = "termtree" version = "0.5.1" @@ -3141,6 +3215,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + [[package]] name = "typetag" version = "0.2.23" diff --git a/temporalio/bridge/Cargo.toml b/temporalio/bridge/Cargo.toml index 4b23ae6ce9..a6bb863c45 100644 --- a/temporalio/bridge/Cargo.toml +++ b/temporalio/bridge/Cargo.toml @@ -72,6 +72,8 @@ temporalio-sdk-core = { version = "0.11.0", path = "./sdk-core/crates/sdk-core", "envconfig", "prometheus", ] } +# The Stream runtime, from the submodule. +temporalio-streams = { path = "./sdk-core/crates/streams" } tokio = "1.26" # Matches the sdk-core client crate's rustls stack; used to build the custom # server certificate verifier for ClientTlsConfig.verification_server_name. diff --git a/temporalio/bridge/_visitor.py b/temporalio/bridge/_visitor.py index 8832b56bc4..54238f8916 100644 --- a/temporalio/bridge/_visitor.py +++ b/temporalio/bridge/_visitor.py @@ -308,6 +308,38 @@ async def _visit_coresdk_workflow_activation_ResolveNexusOperation( if o.HasField("result"): await self._visit_coresdk_nexus_NexusOperationResult(fs, o.result) + async def _visit_temporal_api_stream_v1_StreamRecord( + self, fs: VisitorFunctions, o: Any + ): + if o.HasField("body"): + await self._visit_temporal_api_common_v1_Payload(fs, o.body) + + async def _visit_coresdk_workflow_activation_StreamReadEnd( + self, fs: VisitorFunctions, o: Any + ): + if o.HasField("failure"): + await self._visit_temporal_api_failure_v1_Failure(fs, o.failure) + + async def _visit_coresdk_workflow_activation_ResolveStreamRead( + self, fs: VisitorFunctions, o: Any + ): + for v in o.records: + await self._visit_temporal_api_stream_v1_StreamRecord(fs, v) + if o.HasField("end"): + await self._visit_coresdk_workflow_activation_StreamReadEnd(fs, o.end) + + async def _visit_coresdk_workflow_activation_ResolveStreamAppend( + self, fs: VisitorFunctions, o: Any + ): + if o.HasField("failed"): + await self._visit_temporal_api_failure_v1_Failure(fs, o.failed) + + async def _visit_coresdk_workflow_activation_ResolveStreamClose( + self, fs: VisitorFunctions, o: Any + ): + if o.HasField("failed"): + await self._visit_temporal_api_failure_v1_Failure(fs, o.failed) + async def _visit_coresdk_workflow_activation_WorkflowActivationJob( self, fs: VisitorFunctions, o: Any ): @@ -353,6 +385,18 @@ async def _visit_coresdk_workflow_activation_WorkflowActivationJob( await self._visit_coresdk_workflow_activation_ResolveNexusOperation( fs, o.resolve_nexus_operation ) + elif o.HasField("resolve_stream_read"): + await self._visit_coresdk_workflow_activation_ResolveStreamRead( + fs, o.resolve_stream_read + ) + elif o.HasField("resolve_stream_append"): + await self._visit_coresdk_workflow_activation_ResolveStreamAppend( + fs, o.resolve_stream_append + ) + elif o.HasField("resolve_stream_close"): + await self._visit_coresdk_workflow_activation_ResolveStreamClose( + fs, o.resolve_stream_close + ) async def _visit_coresdk_workflow_activation_WorkflowActivation( self, fs: VisitorFunctions, o: Any @@ -492,6 +536,11 @@ async def _visit_coresdk_workflow_commands_ScheduleNexusOperation( if o.HasField("input"): await self._visit_nexus_operation_input_payload(fs, o.input) + async def _visit_coresdk_workflow_commands_AppendToStream( + self, fs: VisitorFunctions, o: Any + ): + await self._visit_payload_container(fs, o.bodies) + async def _visit_coresdk_workflow_commands_WorkflowCommand( self, fs: VisitorFunctions, o: Any ): @@ -551,6 +600,10 @@ async def _visit_coresdk_workflow_commands_WorkflowCommand( await self._visit_coresdk_workflow_commands_ScheduleNexusOperation( fs, o.schedule_nexus_operation ) + elif o.HasField("append_to_stream"): + await self._visit_coresdk_workflow_commands_AppendToStream( + fs, o.append_to_stream + ) async def _visit_coresdk_workflow_completion_Success( self, fs: VisitorFunctions, o: Any @@ -681,12 +734,6 @@ async def _visit_temporal_api_streamservice_v1_ReadRecordsRequest( ) -> None: pass - async def _visit_temporal_api_stream_v1_StreamRecord( - self, fs: VisitorFunctions, o: Any - ): - if o.HasField("body"): - await self._visit_temporal_api_common_v1_Payload(fs, o.body) - async def _visit_temporal_api_streamservice_v1_ReadRecordsResponse( self, fs: VisitorFunctions, o: Any ): diff --git a/temporalio/bridge/proto/common/__init__.py b/temporalio/bridge/proto/common/__init__.py index a8506090de..b9b0cce027 100644 --- a/temporalio/bridge/proto/common/__init__.py +++ b/temporalio/bridge/proto/common/__init__.py @@ -1,6 +1,7 @@ from .common_pb2 import ( ExternalStorageMetrics, NamespacedWorkflowExecution, + StreamReadStop, VersioningIntent, WorkerDeploymentVersion, ) @@ -8,6 +9,7 @@ __all__ = [ "ExternalStorageMetrics", "NamespacedWorkflowExecution", + "StreamReadStop", "VersioningIntent", "WorkerDeploymentVersion", ] diff --git a/temporalio/bridge/proto/common/common_pb2.py b/temporalio/bridge/proto/common/common_pb2.py index 0b9a5c8169..e492e93c10 100644 --- a/temporalio/bridge/proto/common/common_pb2.py +++ b/temporalio/bridge/proto/common/common_pb2.py @@ -16,7 +16,7 @@ from google.protobuf import duration_pb2 as google_dot_protobuf_dot_duration__pb2 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n%temporal/sdk/core/common/common.proto\x12\x0e\x63oresdk.common\x1a\x1egoogle/protobuf/duration.proto"U\n\x1bNamespacedWorkflowExecution\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t"D\n\x17WorkerDeploymentVersion\x12\x17\n\x0f\x64\x65ployment_name\x18\x01 \x01(\t\x12\x10\n\x08\x62uild_id\x18\x02 \x01(\t"\x92\x01\n\x16\x45xternalStorageMetrics\x12\x15\n\rpayload_count\x18\x01 \x01(\x04\x12\x18\n\x10total_size_bytes\x18\x02 \x01(\x04\x12\x31\n\x0etotal_duration\x18\x03 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x14\n\x0c\x64river_names\x18\x04 \x03(\t*@\n\x10VersioningIntent\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x0e\n\nCOMPATIBLE\x10\x01\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x02\x42,\xea\x02)Temporalio::Internal::Bridge::Api::Commonb\x06proto3' + b'\n%temporal/sdk/core/common/common.proto\x12\x0e\x63oresdk.common\x1a\x1egoogle/protobuf/duration.proto"U\n\x1bNamespacedWorkflowExecution\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t"D\n\x17WorkerDeploymentVersion\x12\x17\n\x0f\x64\x65ployment_name\x18\x01 \x01(\t\x12\x10\n\x08\x62uild_id\x18\x02 \x01(\t"\x92\x01\n\x16\x45xternalStorageMetrics\x12\x15\n\rpayload_count\x18\x01 \x01(\x04\x12\x18\n\x10total_size_bytes\x18\x02 \x01(\x04\x12\x31\n\x0etotal_duration\x18\x03 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x14\n\x0c\x64river_names\x18\x04 \x03(\t*\xc8\x01\n\x0eStreamReadStop\x12 \n\x1cSTREAM_READ_STOP_UNSPECIFIED\x10\x00\x12\x19\n\x15STREAM_READ_STOP_DONE\x10\x01\x12 \n\x1cSTREAM_READ_STOP_MAX_RECORDS\x10\x02\x12\x1e\n\x1aSTREAM_READ_STOP_MAX_BYTES\x10\x03\x12\x19\n\x15STREAM_READ_STOP_IDLE\x10\x04\x12\x1c\n\x18STREAM_READ_STOP_TIMEOUT\x10\x05*@\n\x10VersioningIntent\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x0e\n\nCOMPATIBLE\x10\x01\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x02\x42,\xea\x02)Temporalio::Internal::Bridge::Api::Commonb\x06proto3' ) _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) @@ -26,8 +26,10 @@ DESCRIPTOR._serialized_options = ( b"\352\002)Temporalio::Internal::Bridge::Api::Common" ) - _VERSIONINGINTENT._serialized_start = 395 - _VERSIONINGINTENT._serialized_end = 459 + _STREAMREADSTOP._serialized_start = 396 + _STREAMREADSTOP._serialized_end = 596 + _VERSIONINGINTENT._serialized_start = 598 + _VERSIONINGINTENT._serialized_end = 662 _NAMESPACEDWORKFLOWEXECUTION._serialized_start = 89 _NAMESPACEDWORKFLOWEXECUTION._serialized_end = 174 _WORKERDEPLOYMENTVERSION._serialized_start = 176 diff --git a/temporalio/bridge/proto/common/common_pb2.pyi b/temporalio/bridge/proto/common/common_pb2.pyi index 7c2b057024..da85e098dd 100644 --- a/temporalio/bridge/proto/common/common_pb2.pyi +++ b/temporalio/bridge/proto/common/common_pb2.pyi @@ -21,6 +21,45 @@ else: DESCRIPTOR: google.protobuf.descriptor.FileDescriptor +class _StreamReadStop: + ValueType = typing.NewType("ValueType", builtins.int) + V: typing_extensions.TypeAlias = ValueType + +class _StreamReadStopEnumTypeWrapper( + google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[ + _StreamReadStop.ValueType + ], + builtins.type, +): + DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor + STREAM_READ_STOP_UNSPECIFIED: _StreamReadStop.ValueType # 0 + STREAM_READ_STOP_DONE: _StreamReadStop.ValueType # 1 + """The Stream is closed, and the read gave its last Record.""" + STREAM_READ_STOP_MAX_RECORDS: _StreamReadStop.ValueType # 2 + """The read gave `max_records` Records.""" + STREAM_READ_STOP_MAX_BYTES: _StreamReadStop.ValueType # 3 + """The next Record would pass `max_bytes`.""" + STREAM_READ_STOP_IDLE: _StreamReadStop.ValueType # 4 + """No Record came for `idle_timeout`.""" + STREAM_READ_STOP_TIMEOUT: _StreamReadStop.ValueType # 5 + """`timeout` passed since the read started.""" + +class StreamReadStop(_StreamReadStop, metaclass=_StreamReadStopEnumTypeWrapper): + """Why a read of a Stream ended. A read that failed has no stop.""" + +STREAM_READ_STOP_UNSPECIFIED: StreamReadStop.ValueType # 0 +STREAM_READ_STOP_DONE: StreamReadStop.ValueType # 1 +"""The Stream is closed, and the read gave its last Record.""" +STREAM_READ_STOP_MAX_RECORDS: StreamReadStop.ValueType # 2 +"""The read gave `max_records` Records.""" +STREAM_READ_STOP_MAX_BYTES: StreamReadStop.ValueType # 3 +"""The next Record would pass `max_bytes`.""" +STREAM_READ_STOP_IDLE: StreamReadStop.ValueType # 4 +"""No Record came for `idle_timeout`.""" +STREAM_READ_STOP_TIMEOUT: StreamReadStop.ValueType # 5 +"""`timeout` passed since the read started.""" +global___StreamReadStop = StreamReadStop + class _VersioningIntent: ValueType = typing.NewType("ValueType", builtins.int) V: typing_extensions.TypeAlias = ValueType diff --git a/temporalio/bridge/proto/external_data/__init__.py b/temporalio/bridge/proto/external_data/__init__.py index 836e97ef43..ffe40588eb 100644 --- a/temporalio/bridge/proto/external_data/__init__.py +++ b/temporalio/bridge/proto/external_data/__init__.py @@ -1,6 +1,19 @@ -from .external_data_pb2 import LocalActivityMarkerData, PatchedMarkerData +from .external_data_pb2 import ( + LocalActivityMarkerData, + PatchedMarkerData, + StreamAppendEntry, + StreamCloseEntry, + StreamMarkerData, + StreamReadEnded, + StreamReadEntry, +) __all__ = [ "LocalActivityMarkerData", "PatchedMarkerData", + "StreamAppendEntry", + "StreamCloseEntry", + "StreamMarkerData", + "StreamReadEnded", + "StreamReadEntry", ] diff --git a/temporalio/bridge/proto/external_data/external_data_pb2.py b/temporalio/bridge/proto/external_data/external_data_pb2.py index 4bda6f5144..3e13e28c1f 100644 --- a/temporalio/bridge/proto/external_data/external_data_pb2.py +++ b/temporalio/bridge/proto/external_data/external_data_pb2.py @@ -16,8 +16,18 @@ from google.protobuf import duration_pb2 as google_dot_protobuf_dot_duration__pb2 from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 +from temporalio.api.failure.v1 import ( + message_pb2 as temporal_dot_api_dot_failure_dot_v1_dot_message__pb2, +) +from temporalio.api.stream.v1 import ( + message_pb2 as temporal_dot_api_dot_stream_dot_v1_dot_message__pb2, +) +from temporalio.bridge.proto.common import ( + common_pb2 as temporal_dot_sdk_dot_core_dot_common_dot_common__pb2, +) + DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n3temporal/sdk/core/external_data/external_data.proto\x12\x15\x63oresdk.external_data\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto"\xb2\x02\n\x17LocalActivityMarkerData\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\r\x12\x13\n\x0b\x61\x63tivity_id\x18\x03 \x01(\t\x12\x15\n\ractivity_type\x18\x04 \x01(\t\x12\x31\n\rcomplete_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12*\n\x07\x62\x61\x63koff\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12:\n\x16original_schedule_time\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x1d\n\x10\x61\x63tivation_index\x18\x08 \x01(\x04H\x00\x88\x01\x01\x42\x13\n\x11_activation_index"3\n\x11PatchedMarkerData\x12\n\n\x02id\x18\x01 \x01(\t\x12\x12\n\ndeprecated\x18\x02 \x01(\x08\x42\x32\xea\x02/Temporalio::Internal::Bridge::Api::ExternalDatab\x06proto3' + b'\n3temporal/sdk/core/external_data/external_data.proto\x12\x15\x63oresdk.external_data\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a%temporal/api/failure/v1/message.proto\x1a$temporal/api/stream/v1/message.proto\x1a%temporal/sdk/core/common/common.proto"\xb2\x02\n\x17LocalActivityMarkerData\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\r\x12\x13\n\x0b\x61\x63tivity_id\x18\x03 \x01(\t\x12\x15\n\ractivity_type\x18\x04 \x01(\t\x12\x31\n\rcomplete_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12*\n\x07\x62\x61\x63koff\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12:\n\x16original_schedule_time\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x1d\n\x10\x61\x63tivation_index\x18\x08 \x01(\x04H\x00\x88\x01\x01\x42\x13\n\x11_activation_index"3\n\x11PatchedMarkerData\x12\n\n\x02id\x18\x01 \x01(\t\x12\x12\n\ndeprecated\x18\x02 \x01(\x08"\xf1\x01\n\x10StreamMarkerData\x12\x35\n\x05reads\x18\x01 \x03(\x0b\x32&.coresdk.external_data.StreamReadEntry\x12\x39\n\x07\x61ppends\x18\x02 \x03(\x0b\x32(.coresdk.external_data.StreamAppendEntry\x12\x37\n\x06\x63loses\x18\x03 \x03(\x0b\x32\'.coresdk.external_data.StreamCloseEntry\x12\x32\n\x07streams\x18\x04 \x03(\x0b\x32!.temporal.api.stream.v1.StreamRef"\xae\x02\n\x0fStreamReadEntry\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x14\n\x0cstream_index\x18\x02 \x01(\r\x12\x13\n\x0b\x66rom_offset\x18\x03 \x01(\x0c\x12\x14\n\x0c\x61\x66ter_offset\x18\x04 \x01(\x0c\x12\x13\n\x0blast_offset\x18\x05 \x01(\x0c\x12\r\n\x05\x63ount\x18\x06 \x01(\r\x12\x0e\n\x06\x64igest\x18\x07 \x01(\x0c\x12\x35\n\x07records\x18\x08 \x03(\x0b\x32$.temporal.api.stream.v1.StreamRecord\x12\x13\n\x0bpage_counts\x18\t \x03(\r\x12\x18\n\x10page_activations\x18\n \x03(\r\x12\x33\n\x03\x65nd\x18\x0b \x01(\x0b\x32&.coresdk.external_data.StreamReadEnded"\xa9\x01\n\x0fStreamReadEnded\x12\x13\n\x0blast_offset\x18\x01 \x01(\x0c\x12,\n\x04stop\x18\x02 \x01(\x0e\x32\x1e.coresdk.common.StreamReadStop\x12\x0c\n\x04\x64one\x18\x03 \x01(\x08\x12\x31\n\x07\x66\x61ilure\x18\x04 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x12\n\nactivation\x18\x05 \x01(\r"\xa8\x01\n\x11StreamAppendEntry\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x14\n\x0cstream_index\x18\x02 \x01(\r\x12\x14\n\x0c\x66irst_offset\x18\x03 \x01(\x0c\x12\x13\n\x0blast_offset\x18\x04 \x01(\x0c\x12\x31\n\x07\x66\x61ilure\x18\x05 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x12\n\nactivation\x18\x06 \x01(\r"\xcf\x01\n\x10StreamCloseEntry\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x14\n\x0cstream_index\x18\x02 \x01(\r\x12?\n\x0cintent_epoch\x18\x03 \x01(\x0b\x32).temporal.api.stream.v1.WorkflowTaskEpoch\x12\x10\n\x08resolved\x18\x04 \x01(\x08\x12\x31\n\x07\x66\x61ilure\x18\x05 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x12\n\nactivation\x18\x06 \x01(\rB2\xea\x02/Temporalio::Internal::Bridge::Api::ExternalDatab\x06proto3' ) _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) @@ -27,8 +37,18 @@ DESCRIPTOR._serialized_options = ( b"\352\002/Temporalio::Internal::Bridge::Api::ExternalData" ) - _LOCALACTIVITYMARKERDATA._serialized_start = 144 - _LOCALACTIVITYMARKERDATA._serialized_end = 450 - _PATCHEDMARKERDATA._serialized_start = 452 - _PATCHEDMARKERDATA._serialized_end = 503 + _LOCALACTIVITYMARKERDATA._serialized_start = 260 + _LOCALACTIVITYMARKERDATA._serialized_end = 566 + _PATCHEDMARKERDATA._serialized_start = 568 + _PATCHEDMARKERDATA._serialized_end = 619 + _STREAMMARKERDATA._serialized_start = 622 + _STREAMMARKERDATA._serialized_end = 863 + _STREAMREADENTRY._serialized_start = 866 + _STREAMREADENTRY._serialized_end = 1168 + _STREAMREADENDED._serialized_start = 1171 + _STREAMREADENDED._serialized_end = 1340 + _STREAMAPPENDENTRY._serialized_start = 1343 + _STREAMAPPENDENTRY._serialized_end = 1511 + _STREAMCLOSEENTRY._serialized_start = 1514 + _STREAMCLOSEENTRY._serialized_end = 1721 # @@protoc_insertion_point(module_scope) diff --git a/temporalio/bridge/proto/external_data/external_data_pb2.pyi b/temporalio/bridge/proto/external_data/external_data_pb2.pyi index edcd4581a1..61b5009772 100644 --- a/temporalio/bridge/proto/external_data/external_data_pb2.pyi +++ b/temporalio/bridge/proto/external_data/external_data_pb2.pyi @@ -4,13 +4,19 @@ isort:skip_file """ import builtins +import collections.abc import typing import google.protobuf.descriptor import google.protobuf.duration_pb2 +import google.protobuf.internal.containers import google.protobuf.message import google.protobuf.timestamp_pb2 +import temporalio.api.failure.v1.message_pb2 +import temporalio.api.stream.v1.message_pb2 +import temporalio.bridge.proto.common.common_pb2 + DESCRIPTOR: google.protobuf.descriptor.FileDescriptor @typing.final @@ -139,3 +145,340 @@ class PatchedMarkerData(google.protobuf.message.Message): ) -> None: ... global___PatchedMarkerData = PatchedMarkerData + +@typing.final +class StreamMarkerData(google.protobuf.message.Message): + """The data of the marker `core_stream`, which Core records once for each workflow task in which + the workflow got a Stream result. It is in the detail `data` as `binary/protobuf`, so that a + marker with Records stays small. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + READS_FIELD_NUMBER: builtins.int + APPENDS_FIELD_NUMBER: builtins.int + CLOSES_FIELD_NUMBER: builtins.int + STREAMS_FIELD_NUMBER: builtins.int + @property + def reads( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + global___StreamReadEntry + ]: ... + @property + def appends( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + global___StreamAppendEntry + ]: ... + @property + def closes( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + global___StreamCloseEntry + ]: ... + @property + def streams( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + temporalio.api.stream.v1.message_pb2.StreamRef + ]: + """Each Stream of the entries, once. An entry gives the index of its Stream here, so that a task + with many results on one Stream holds its ref once. + """ + + def __init__( + self, + *, + reads: collections.abc.Iterable[global___StreamReadEntry] | None = ..., + appends: collections.abc.Iterable[global___StreamAppendEntry] | None = ..., + closes: collections.abc.Iterable[global___StreamCloseEntry] | None = ..., + streams: collections.abc.Iterable[ + temporalio.api.stream.v1.message_pb2.StreamRef + ] + | None = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing.Literal[ + "appends", + b"appends", + "closes", + b"closes", + "reads", + b"reads", + "streams", + b"streams", + ], + ) -> None: ... + +global___StreamMarkerData = StreamMarkerData + +@typing.final +class StreamReadEntry(google.protobuf.message.Message): + """What one read gave the workflow in one workflow task.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SEQ_FIELD_NUMBER: builtins.int + STREAM_INDEX_FIELD_NUMBER: builtins.int + FROM_OFFSET_FIELD_NUMBER: builtins.int + AFTER_OFFSET_FIELD_NUMBER: builtins.int + LAST_OFFSET_FIELD_NUMBER: builtins.int + COUNT_FIELD_NUMBER: builtins.int + DIGEST_FIELD_NUMBER: builtins.int + RECORDS_FIELD_NUMBER: builtins.int + PAGE_COUNTS_FIELD_NUMBER: builtins.int + PAGE_ACTIVATIONS_FIELD_NUMBER: builtins.int + END_FIELD_NUMBER: builtins.int + seq: builtins.int + stream_index: builtins.int + """The index of the read's Stream in `StreamMarkerData.streams`.""" + from_offset: builtins.bytes + """The `from_offset` of the read, so a replay can check that the read is the same.""" + after_offset: builtins.bytes + """The offset before the first Record of this task. A replay reads after it.""" + last_offset: builtins.bytes + """The offset of the last Record of this task. Empty when the task got no Record.""" + count: builtins.int + """The number of Records of this task.""" + digest: builtins.bytes + """The digest of the Records of this task. Empty when `records` holds them.""" + @property + def records( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + temporalio.api.stream.v1.message_pb2.StreamRecord + ]: + """The Records of this task, only for a read with its Records in history.""" + + @property + def page_counts( + self, + ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.int]: + """The number of Records of each page, in order. A page ends where its count ends.""" + + @property + def page_activations( + self, + ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.int]: + """The activation of each page in the task. The first is the index, and each later one is the + gap to the one before. + """ + + @property + def end(self) -> global___StreamReadEnded: + """Set when the read ended in this task.""" + + def __init__( + self, + *, + seq: builtins.int = ..., + stream_index: builtins.int = ..., + from_offset: builtins.bytes = ..., + after_offset: builtins.bytes = ..., + last_offset: builtins.bytes = ..., + count: builtins.int = ..., + digest: builtins.bytes = ..., + records: collections.abc.Iterable[ + temporalio.api.stream.v1.message_pb2.StreamRecord + ] + | None = ..., + page_counts: collections.abc.Iterable[builtins.int] | None = ..., + page_activations: collections.abc.Iterable[builtins.int] | None = ..., + end: global___StreamReadEnded | None = ..., + ) -> None: ... + def HasField(self, field_name: typing.Literal["end", b"end"]) -> builtins.bool: ... + def ClearField( + self, + field_name: typing.Literal[ + "after_offset", + b"after_offset", + "count", + b"count", + "digest", + b"digest", + "end", + b"end", + "from_offset", + b"from_offset", + "last_offset", + b"last_offset", + "page_activations", + b"page_activations", + "page_counts", + b"page_counts", + "records", + b"records", + "seq", + b"seq", + "stream_index", + b"stream_index", + ], + ) -> None: ... + +global___StreamReadEntry = StreamReadEntry + +@typing.final +class StreamReadEnded(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + LAST_OFFSET_FIELD_NUMBER: builtins.int + STOP_FIELD_NUMBER: builtins.int + DONE_FIELD_NUMBER: builtins.int + FAILURE_FIELD_NUMBER: builtins.int + ACTIVATION_FIELD_NUMBER: builtins.int + last_offset: builtins.bytes + stop: temporalio.bridge.proto.common.common_pb2.StreamReadStop.ValueType + done: builtins.bool + activation: builtins.int + """The activation in the task that got the end.""" + @property + def failure(self) -> temporalio.api.failure.v1.message_pb2.Failure: ... + def __init__( + self, + *, + last_offset: builtins.bytes = ..., + stop: temporalio.bridge.proto.common.common_pb2.StreamReadStop.ValueType = ..., + done: builtins.bool = ..., + failure: temporalio.api.failure.v1.message_pb2.Failure | None = ..., + activation: builtins.int = ..., + ) -> None: ... + def HasField( + self, field_name: typing.Literal["failure", b"failure"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing.Literal[ + "activation", + b"activation", + "done", + b"done", + "failure", + b"failure", + "last_offset", + b"last_offset", + "stop", + b"stop", + ], + ) -> None: ... + +global___StreamReadEnded = StreamReadEnded + +@typing.final +class StreamAppendEntry(google.protobuf.message.Message): + """The result of one append, in the workflow task whose activation got it.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SEQ_FIELD_NUMBER: builtins.int + STREAM_INDEX_FIELD_NUMBER: builtins.int + FIRST_OFFSET_FIELD_NUMBER: builtins.int + LAST_OFFSET_FIELD_NUMBER: builtins.int + FAILURE_FIELD_NUMBER: builtins.int + ACTIVATION_FIELD_NUMBER: builtins.int + seq: builtins.int + stream_index: builtins.int + """The index of the append's Stream in `StreamMarkerData.streams`.""" + first_offset: builtins.bytes + last_offset: builtins.bytes + activation: builtins.int + """The activation in the task that got the result.""" + @property + def failure(self) -> temporalio.api.failure.v1.message_pb2.Failure: ... + def __init__( + self, + *, + seq: builtins.int = ..., + stream_index: builtins.int = ..., + first_offset: builtins.bytes = ..., + last_offset: builtins.bytes = ..., + failure: temporalio.api.failure.v1.message_pb2.Failure | None = ..., + activation: builtins.int = ..., + ) -> None: ... + def HasField( + self, field_name: typing.Literal["failure", b"failure"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing.Literal[ + "activation", + b"activation", + "failure", + b"failure", + "first_offset", + b"first_offset", + "last_offset", + b"last_offset", + "seq", + b"seq", + "stream_index", + b"stream_index", + ], + ) -> None: ... + +global___StreamAppendEntry = StreamAppendEntry + +@typing.final +class StreamCloseEntry(google.protobuf.message.Message): + """A close in one workflow task: its intent, in the task that asked for it, or its result, in the + task whose activation got it. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SEQ_FIELD_NUMBER: builtins.int + STREAM_INDEX_FIELD_NUMBER: builtins.int + INTENT_EPOCH_FIELD_NUMBER: builtins.int + RESOLVED_FIELD_NUMBER: builtins.int + FAILURE_FIELD_NUMBER: builtins.int + ACTIVATION_FIELD_NUMBER: builtins.int + seq: builtins.int + stream_index: builtins.int + """The index of the closed Stream in `StreamMarkerData.streams`.""" + resolved: builtins.bool + """True on the result. `failure` is unset when the Stream is closed.""" + activation: builtins.int + """The activation in the task that got the result.""" + @property + def intent_epoch(self) -> temporalio.api.stream.v1.message_pb2.WorkflowTaskEpoch: + """Set on the intent, the epoch that Core closes with after the server took the task.""" + + @property + def failure(self) -> temporalio.api.failure.v1.message_pb2.Failure: ... + def __init__( + self, + *, + seq: builtins.int = ..., + stream_index: builtins.int = ..., + intent_epoch: temporalio.api.stream.v1.message_pb2.WorkflowTaskEpoch + | None = ..., + resolved: builtins.bool = ..., + failure: temporalio.api.failure.v1.message_pb2.Failure | None = ..., + activation: builtins.int = ..., + ) -> None: ... + def HasField( + self, + field_name: typing.Literal[ + "failure", b"failure", "intent_epoch", b"intent_epoch" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing.Literal[ + "activation", + b"activation", + "failure", + b"failure", + "intent_epoch", + b"intent_epoch", + "resolved", + b"resolved", + "seq", + b"seq", + "stream_index", + b"stream_index", + ], + ) -> None: ... + +global___StreamCloseEntry = StreamCloseEntry diff --git a/temporalio/bridge/proto/workflow_activation/__init__.py b/temporalio/bridge/proto/workflow_activation/__init__.py index 5c178e2185..4e459cc9b5 100644 --- a/temporalio/bridge/proto/workflow_activation/__init__.py +++ b/temporalio/bridge/proto/workflow_activation/__init__.py @@ -16,7 +16,12 @@ ResolveNexusOperationStart, ResolveRequestCancelExternalWorkflow, ResolveSignalExternalWorkflow, + ResolveStreamAppend, + ResolveStreamClose, + ResolveStreamRead, SignalWorkflow, + StreamAppended, + StreamReadEnd, UpdateRandomSeed, WorkflowActivation, WorkflowActivationJob, @@ -40,7 +45,12 @@ "ResolveNexusOperationStart", "ResolveRequestCancelExternalWorkflow", "ResolveSignalExternalWorkflow", + "ResolveStreamAppend", + "ResolveStreamClose", + "ResolveStreamRead", "SignalWorkflow", + "StreamAppended", + "StreamReadEnd", "UpdateRandomSeed", "WorkflowActivation", "WorkflowActivationJob", diff --git a/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.py b/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.py index 589119c72b..863fbc082a 100644 --- a/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.py +++ b/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.py @@ -32,6 +32,9 @@ from temporalio.api.nexus.v1 import ( message_pb2 as temporal_dot_api_dot_nexus_dot_v1_dot_message__pb2, ) +from temporalio.api.stream.v1 import ( + message_pb2 as temporal_dot_api_dot_stream_dot_v1_dot_message__pb2, +) from temporalio.api.update.v1 import ( message_pb2 as temporal_dot_api_dot_update_dot_v1_dot_message__pb2, ) @@ -49,7 +52,7 @@ ) DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n?temporal/sdk/core/workflow_activation/workflow_activation.proto\x12\x1b\x63oresdk.workflow_activation\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a%temporal/api/failure/v1/message.proto\x1a#temporal/api/nexus/v1/message.proto\x1a$temporal/api/update/v1/message.proto\x1a$temporal/api/common/v1/message.proto\x1a(temporal/api/enums/v1/failed_cause.proto\x1a$temporal/api/enums/v1/workflow.proto\x1a\x37temporal/sdk/core/activity_result/activity_result.proto\x1a\x35temporal/sdk/core/child_workflow/child_workflow.proto\x1a%temporal/sdk/core/common/common.proto\x1a#temporal/sdk/core/nexus/nexus.proto"\xa2\x04\n\x12WorkflowActivation\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12-\n\ttimestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x14\n\x0cis_replaying\x18\x03 \x01(\x08\x12\x16\n\x0ehistory_length\x18\x04 \x01(\r\x12@\n\x04jobs\x18\x05 \x03(\x0b\x32\x32.coresdk.workflow_activation.WorkflowActivationJob\x12 \n\x18\x61vailable_internal_flags\x18\x06 \x03(\r\x12\x1a\n\x12history_size_bytes\x18\x07 \x01(\x04\x12!\n\x19\x63ontinue_as_new_suggested\x18\x08 \x01(\x08\x12T\n#deployment_version_for_current_task\x18\t \x01(\x0b\x32\'.coresdk.common.WorkerDeploymentVersion\x12\x18\n\x10last_sdk_version\x18\n \x01(\t\x12Z\n\x1fsuggest_continue_as_new_reasons\x18\x0b \x03(\x0e\x32\x31.temporal.api.enums.v1.SuggestContinueAsNewReason\x12\x30\n(target_worker_deployment_version_changed\x18\x0c \x01(\x08"\xe0\n\n\x15WorkflowActivationJob\x12N\n\x13initialize_workflow\x18\x01 \x01(\x0b\x32/.coresdk.workflow_activation.InitializeWorkflowH\x00\x12<\n\nfire_timer\x18\x02 \x01(\x0b\x32&.coresdk.workflow_activation.FireTimerH\x00\x12K\n\x12update_random_seed\x18\x04 \x01(\x0b\x32-.coresdk.workflow_activation.UpdateRandomSeedH\x00\x12\x44\n\x0equery_workflow\x18\x05 \x01(\x0b\x32*.coresdk.workflow_activation.QueryWorkflowH\x00\x12\x46\n\x0f\x63\x61ncel_workflow\x18\x06 \x01(\x0b\x32+.coresdk.workflow_activation.CancelWorkflowH\x00\x12\x46\n\x0fsignal_workflow\x18\x07 \x01(\x0b\x32+.coresdk.workflow_activation.SignalWorkflowH\x00\x12H\n\x10resolve_activity\x18\x08 \x01(\x0b\x32,.coresdk.workflow_activation.ResolveActivityH\x00\x12G\n\x10notify_has_patch\x18\t \x01(\x0b\x32+.coresdk.workflow_activation.NotifyHasPatchH\x00\x12q\n&resolve_child_workflow_execution_start\x18\n \x01(\x0b\x32?.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartH\x00\x12\x66\n resolve_child_workflow_execution\x18\x0b \x01(\x0b\x32:.coresdk.workflow_activation.ResolveChildWorkflowExecutionH\x00\x12\x66\n resolve_signal_external_workflow\x18\x0c \x01(\x0b\x32:.coresdk.workflow_activation.ResolveSignalExternalWorkflowH\x00\x12u\n(resolve_request_cancel_external_workflow\x18\r \x01(\x0b\x32\x41.coresdk.workflow_activation.ResolveRequestCancelExternalWorkflowH\x00\x12:\n\tdo_update\x18\x0e \x01(\x0b\x32%.coresdk.workflow_activation.DoUpdateH\x00\x12`\n\x1dresolve_nexus_operation_start\x18\x0f \x01(\x0b\x32\x37.coresdk.workflow_activation.ResolveNexusOperationStartH\x00\x12U\n\x17resolve_nexus_operation\x18\x10 \x01(\x0b\x32\x32.coresdk.workflow_activation.ResolveNexusOperationH\x00\x12I\n\x11remove_from_cache\x18\x32 \x01(\x0b\x32,.coresdk.workflow_activation.RemoveFromCacheH\x00\x42\t\n\x07variant"\xe3\x0b\n\x12InitializeWorkflow\x12\x15\n\rworkflow_type\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x32\n\targuments\x18\x03 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x17\n\x0frandomness_seed\x18\x04 \x01(\x04\x12M\n\x07headers\x18\x05 \x03(\x0b\x32<.coresdk.workflow_activation.InitializeWorkflow.HeadersEntry\x12\x10\n\x08identity\x18\x06 \x01(\t\x12I\n\x14parent_workflow_info\x18\x07 \x01(\x0b\x32+.coresdk.common.NamespacedWorkflowExecution\x12=\n\x1aworkflow_execution_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x37\n\x14workflow_run_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\'\n\x1f\x63ontinued_from_execution_run_id\x18\x0b \x01(\t\x12J\n\x13\x63ontinued_initiator\x18\x0c \x01(\x0e\x32-.temporal.api.enums.v1.ContinueAsNewInitiator\x12;\n\x11\x63ontinued_failure\x18\r \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12@\n\x16last_completion_result\x18\x0e \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x1e\n\x16\x66irst_execution_run_id\x18\x0f \x01(\t\x12\x39\n\x0cretry_policy\x18\x10 \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x0f\n\x07\x61ttempt\x18\x11 \x01(\x05\x12\x15\n\rcron_schedule\x18\x12 \x01(\t\x12\x46\n"workflow_execution_expiration_time\x18\x13 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x45\n"cron_schedule_to_schedule_interval\x18\x14 \x01(\x0b\x32\x19.google.protobuf.Duration\x12*\n\x04memo\x18\x15 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo\x12\x43\n\x11search_attributes\x18\x16 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12.\n\nstart_time\x18\x17 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12@\n\rroot_workflow\x18\x18 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x32\n\x08priority\x18\x19 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12!\n\x19original_execution_run_id\x18\x1a \x01(\t\x12\x65\n&propagated_nexus_serialization_context\x18\x1b \x01(\x0b\x32\x35.temporal.api.nexus.v1.PropagatedSerializationContext\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x18\n\tFireTimer\x12\x0b\n\x03seq\x18\x01 \x01(\r"m\n\x0fResolveActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12;\n\x06result\x18\x02 \x01(\x0b\x32+.coresdk.activity_result.ActivityResolution\x12\x10\n\x08is_local\x18\x03 \x01(\x08"\xd1\x02\n"ResolveChildWorkflowExecutionStart\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12[\n\tsucceeded\x18\x02 \x01(\x0b\x32\x46.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartSuccessH\x00\x12X\n\x06\x66\x61iled\x18\x03 \x01(\x0b\x32\x46.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartFailureH\x00\x12]\n\tcancelled\x18\x04 \x01(\x0b\x32H.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartCancelledH\x00\x42\x08\n\x06status";\n)ResolveChildWorkflowExecutionStartSuccess\x12\x0e\n\x06run_id\x18\x01 \x01(\t"\xa6\x01\n)ResolveChildWorkflowExecutionStartFailure\x12\x13\n\x0bworkflow_id\x18\x01 \x01(\t\x12\x15\n\rworkflow_type\x18\x02 \x01(\t\x12M\n\x05\x63\x61use\x18\x03 \x01(\x0e\x32>.coresdk.child_workflow.StartChildWorkflowExecutionFailedCause"`\n+ResolveChildWorkflowExecutionStartCancelled\x12\x31\n\x07\x66\x61ilure\x18\x01 \x01(\x0b\x32 .temporal.api.failure.v1.Failure"i\n\x1dResolveChildWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12;\n\x06result\x18\x02 \x01(\x0b\x32+.coresdk.child_workflow.ChildWorkflowResult"+\n\x10UpdateRandomSeed\x12\x17\n\x0frandomness_seed\x18\x01 \x01(\x04"\x84\x02\n\rQueryWorkflow\x12\x10\n\x08query_id\x18\x01 \x01(\t\x12\x12\n\nquery_type\x18\x02 \x01(\t\x12\x32\n\targuments\x18\x03 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12H\n\x07headers\x18\x05 \x03(\x0b\x32\x37.coresdk.workflow_activation.QueryWorkflow.HeadersEntry\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01" \n\x0e\x43\x61ncelWorkflow\x12\x0e\n\x06reason\x18\x01 \x01(\t"\xa1\x02\n\x0eSignalWorkflow\x12\x13\n\x0bsignal_name\x18\x01 \x01(\t\x12.\n\x05input\x18\x02 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x10\n\x08identity\x18\x03 \x01(\t\x12I\n\x07headers\x18\x05 \x03(\x0b\x32\x38.coresdk.workflow_activation.SignalWorkflow.HeadersEntry\x12\x1c\n\x14originating_event_id\x18\x06 \x01(\x03\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01""\n\x0eNotifyHasPatch\x12\x10\n\x08patch_id\x18\x01 \x01(\t"\xb1\x01\n\x1dResolveSignalExternalWorkflow\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x07\x66\x61ilure\x18\x02 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12P\n\x05\x63\x61use\x18\x03 \x01(\x0e\x32\x41.temporal.api.enums.v1.SignalExternalWorkflowExecutionFailedCause"\xb8\x01\n$ResolveRequestCancelExternalWorkflow\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x07\x66\x61ilure\x18\x02 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12P\n\x05\x63\x61use\x18\x03 \x01(\x0e\x32\x41.temporal.api.enums.v1.CancelExternalWorkflowExecutionFailedCause"\xb2\x03\n\x08\x44oUpdate\x12\n\n\x02id\x18\x01 \x01(\t\x12\x1c\n\x14protocol_instance_id\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12.\n\x05input\x18\x04 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x43\n\x07headers\x18\x05 \x03(\x0b\x32\x32.coresdk.workflow_activation.DoUpdate.HeadersEntry\x12*\n\x04meta\x18\x06 \x01(\x0b\x32\x1c.temporal.api.update.v1.Meta\x12\x15\n\rrun_validator\x18\x07 \x01(\x08\x12\x65\n&propagated_nexus_serialization_context\x18\x08 \x01(\x0b\x32\x35.temporal.api.nexus.v1.PropagatedSerializationContext\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x9a\x01\n\x1aResolveNexusOperationStart\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x19\n\x0foperation_token\x18\x02 \x01(\tH\x00\x12\x16\n\x0cstarted_sync\x18\x03 \x01(\x08H\x00\x12\x32\n\x06\x66\x61iled\x18\x04 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x42\x08\n\x06status"Y\n\x15ResolveNexusOperation\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x33\n\x06result\x18\x02 \x01(\x0b\x32#.coresdk.nexus.NexusOperationResult"\xe0\x02\n\x0fRemoveFromCache\x12\x0f\n\x07message\x18\x01 \x01(\t\x12K\n\x06reason\x18\x02 \x01(\x0e\x32;.coresdk.workflow_activation.RemoveFromCache.EvictionReason"\xee\x01\n\x0e\x45victionReason\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x0e\n\nCACHE_FULL\x10\x01\x12\x0e\n\nCACHE_MISS\x10\x02\x12\x12\n\x0eNONDETERMINISM\x10\x03\x12\r\n\tLANG_FAIL\x10\x04\x12\x12\n\x0eLANG_REQUESTED\x10\x05\x12\x12\n\x0eTASK_NOT_FOUND\x10\x06\x12\x15\n\x11UNHANDLED_COMMAND\x10\x07\x12\t\n\x05\x46\x41TAL\x10\x08\x12\x1f\n\x1bPAGINATION_OR_HISTORY_FETCH\x10\t\x12\x1d\n\x19WORKFLOW_EXECUTION_ENDING\x10\nB8\xea\x02\x35Temporalio::Internal::Bridge::Api::WorkflowActivationb\x06proto3' + b'\n?temporal/sdk/core/workflow_activation/workflow_activation.proto\x12\x1b\x63oresdk.workflow_activation\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a%temporal/api/failure/v1/message.proto\x1a#temporal/api/nexus/v1/message.proto\x1a$temporal/api/update/v1/message.proto\x1a$temporal/api/common/v1/message.proto\x1a$temporal/api/stream/v1/message.proto\x1a(temporal/api/enums/v1/failed_cause.proto\x1a$temporal/api/enums/v1/workflow.proto\x1a\x37temporal/sdk/core/activity_result/activity_result.proto\x1a\x35temporal/sdk/core/child_workflow/child_workflow.proto\x1a%temporal/sdk/core/common/common.proto\x1a#temporal/sdk/core/nexus/nexus.proto"\xa2\x04\n\x12WorkflowActivation\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12-\n\ttimestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x14\n\x0cis_replaying\x18\x03 \x01(\x08\x12\x16\n\x0ehistory_length\x18\x04 \x01(\r\x12@\n\x04jobs\x18\x05 \x03(\x0b\x32\x32.coresdk.workflow_activation.WorkflowActivationJob\x12 \n\x18\x61vailable_internal_flags\x18\x06 \x03(\r\x12\x1a\n\x12history_size_bytes\x18\x07 \x01(\x04\x12!\n\x19\x63ontinue_as_new_suggested\x18\x08 \x01(\x08\x12T\n#deployment_version_for_current_task\x18\t \x01(\x0b\x32\'.coresdk.common.WorkerDeploymentVersion\x12\x18\n\x10last_sdk_version\x18\n \x01(\t\x12Z\n\x1fsuggest_continue_as_new_reasons\x18\x0b \x03(\x0e\x32\x31.temporal.api.enums.v1.SuggestContinueAsNewReason\x12\x30\n(target_worker_deployment_version_changed\x18\x0c \x01(\x08"\xd3\x0c\n\x15WorkflowActivationJob\x12N\n\x13initialize_workflow\x18\x01 \x01(\x0b\x32/.coresdk.workflow_activation.InitializeWorkflowH\x00\x12<\n\nfire_timer\x18\x02 \x01(\x0b\x32&.coresdk.workflow_activation.FireTimerH\x00\x12K\n\x12update_random_seed\x18\x04 \x01(\x0b\x32-.coresdk.workflow_activation.UpdateRandomSeedH\x00\x12\x44\n\x0equery_workflow\x18\x05 \x01(\x0b\x32*.coresdk.workflow_activation.QueryWorkflowH\x00\x12\x46\n\x0f\x63\x61ncel_workflow\x18\x06 \x01(\x0b\x32+.coresdk.workflow_activation.CancelWorkflowH\x00\x12\x46\n\x0fsignal_workflow\x18\x07 \x01(\x0b\x32+.coresdk.workflow_activation.SignalWorkflowH\x00\x12H\n\x10resolve_activity\x18\x08 \x01(\x0b\x32,.coresdk.workflow_activation.ResolveActivityH\x00\x12G\n\x10notify_has_patch\x18\t \x01(\x0b\x32+.coresdk.workflow_activation.NotifyHasPatchH\x00\x12q\n&resolve_child_workflow_execution_start\x18\n \x01(\x0b\x32?.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartH\x00\x12\x66\n resolve_child_workflow_execution\x18\x0b \x01(\x0b\x32:.coresdk.workflow_activation.ResolveChildWorkflowExecutionH\x00\x12\x66\n resolve_signal_external_workflow\x18\x0c \x01(\x0b\x32:.coresdk.workflow_activation.ResolveSignalExternalWorkflowH\x00\x12u\n(resolve_request_cancel_external_workflow\x18\r \x01(\x0b\x32\x41.coresdk.workflow_activation.ResolveRequestCancelExternalWorkflowH\x00\x12:\n\tdo_update\x18\x0e \x01(\x0b\x32%.coresdk.workflow_activation.DoUpdateH\x00\x12`\n\x1dresolve_nexus_operation_start\x18\x0f \x01(\x0b\x32\x37.coresdk.workflow_activation.ResolveNexusOperationStartH\x00\x12U\n\x17resolve_nexus_operation\x18\x10 \x01(\x0b\x32\x32.coresdk.workflow_activation.ResolveNexusOperationH\x00\x12M\n\x13resolve_stream_read\x18\x11 \x01(\x0b\x32..coresdk.workflow_activation.ResolveStreamReadH\x00\x12Q\n\x15resolve_stream_append\x18\x12 \x01(\x0b\x32\x30.coresdk.workflow_activation.ResolveStreamAppendH\x00\x12O\n\x14resolve_stream_close\x18\x13 \x01(\x0b\x32/.coresdk.workflow_activation.ResolveStreamCloseH\x00\x12I\n\x11remove_from_cache\x18\x32 \x01(\x0b\x32,.coresdk.workflow_activation.RemoveFromCacheH\x00\x42\t\n\x07variant"\xe3\x0b\n\x12InitializeWorkflow\x12\x15\n\rworkflow_type\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x32\n\targuments\x18\x03 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x17\n\x0frandomness_seed\x18\x04 \x01(\x04\x12M\n\x07headers\x18\x05 \x03(\x0b\x32<.coresdk.workflow_activation.InitializeWorkflow.HeadersEntry\x12\x10\n\x08identity\x18\x06 \x01(\t\x12I\n\x14parent_workflow_info\x18\x07 \x01(\x0b\x32+.coresdk.common.NamespacedWorkflowExecution\x12=\n\x1aworkflow_execution_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x37\n\x14workflow_run_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\'\n\x1f\x63ontinued_from_execution_run_id\x18\x0b \x01(\t\x12J\n\x13\x63ontinued_initiator\x18\x0c \x01(\x0e\x32-.temporal.api.enums.v1.ContinueAsNewInitiator\x12;\n\x11\x63ontinued_failure\x18\r \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12@\n\x16last_completion_result\x18\x0e \x01(\x0b\x32 .temporal.api.common.v1.Payloads\x12\x1e\n\x16\x66irst_execution_run_id\x18\x0f \x01(\t\x12\x39\n\x0cretry_policy\x18\x10 \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x0f\n\x07\x61ttempt\x18\x11 \x01(\x05\x12\x15\n\rcron_schedule\x18\x12 \x01(\t\x12\x46\n"workflow_execution_expiration_time\x18\x13 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x45\n"cron_schedule_to_schedule_interval\x18\x14 \x01(\x0b\x32\x19.google.protobuf.Duration\x12*\n\x04memo\x18\x15 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo\x12\x43\n\x11search_attributes\x18\x16 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12.\n\nstart_time\x18\x17 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12@\n\rroot_workflow\x18\x18 \x01(\x0b\x32).temporal.api.common.v1.WorkflowExecution\x12\x32\n\x08priority\x18\x19 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12!\n\x19original_execution_run_id\x18\x1a \x01(\t\x12\x65\n&propagated_nexus_serialization_context\x18\x1b \x01(\x0b\x32\x35.temporal.api.nexus.v1.PropagatedSerializationContext\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x18\n\tFireTimer\x12\x0b\n\x03seq\x18\x01 \x01(\r"m\n\x0fResolveActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12;\n\x06result\x18\x02 \x01(\x0b\x32+.coresdk.activity_result.ActivityResolution\x12\x10\n\x08is_local\x18\x03 \x01(\x08"\xd1\x02\n"ResolveChildWorkflowExecutionStart\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12[\n\tsucceeded\x18\x02 \x01(\x0b\x32\x46.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartSuccessH\x00\x12X\n\x06\x66\x61iled\x18\x03 \x01(\x0b\x32\x46.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartFailureH\x00\x12]\n\tcancelled\x18\x04 \x01(\x0b\x32H.coresdk.workflow_activation.ResolveChildWorkflowExecutionStartCancelledH\x00\x42\x08\n\x06status";\n)ResolveChildWorkflowExecutionStartSuccess\x12\x0e\n\x06run_id\x18\x01 \x01(\t"\xa6\x01\n)ResolveChildWorkflowExecutionStartFailure\x12\x13\n\x0bworkflow_id\x18\x01 \x01(\t\x12\x15\n\rworkflow_type\x18\x02 \x01(\t\x12M\n\x05\x63\x61use\x18\x03 \x01(\x0e\x32>.coresdk.child_workflow.StartChildWorkflowExecutionFailedCause"`\n+ResolveChildWorkflowExecutionStartCancelled\x12\x31\n\x07\x66\x61ilure\x18\x01 \x01(\x0b\x32 .temporal.api.failure.v1.Failure"i\n\x1dResolveChildWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12;\n\x06result\x18\x02 \x01(\x0b\x32+.coresdk.child_workflow.ChildWorkflowResult"+\n\x10UpdateRandomSeed\x12\x17\n\x0frandomness_seed\x18\x01 \x01(\x04"\x84\x02\n\rQueryWorkflow\x12\x10\n\x08query_id\x18\x01 \x01(\t\x12\x12\n\nquery_type\x18\x02 \x01(\t\x12\x32\n\targuments\x18\x03 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12H\n\x07headers\x18\x05 \x03(\x0b\x32\x37.coresdk.workflow_activation.QueryWorkflow.HeadersEntry\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01" \n\x0e\x43\x61ncelWorkflow\x12\x0e\n\x06reason\x18\x01 \x01(\t"\xa1\x02\n\x0eSignalWorkflow\x12\x13\n\x0bsignal_name\x18\x01 \x01(\t\x12.\n\x05input\x18\x02 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x10\n\x08identity\x18\x03 \x01(\t\x12I\n\x07headers\x18\x05 \x03(\x0b\x32\x38.coresdk.workflow_activation.SignalWorkflow.HeadersEntry\x12\x1c\n\x14originating_event_id\x18\x06 \x01(\x03\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01""\n\x0eNotifyHasPatch\x12\x10\n\x08patch_id\x18\x01 \x01(\t"\xb1\x01\n\x1dResolveSignalExternalWorkflow\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x07\x66\x61ilure\x18\x02 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12P\n\x05\x63\x61use\x18\x03 \x01(\x0e\x32\x41.temporal.api.enums.v1.SignalExternalWorkflowExecutionFailedCause"\xb8\x01\n$ResolveRequestCancelExternalWorkflow\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x07\x66\x61ilure\x18\x02 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12P\n\x05\x63\x61use\x18\x03 \x01(\x0e\x32\x41.temporal.api.enums.v1.CancelExternalWorkflowExecutionFailedCause"\xb2\x03\n\x08\x44oUpdate\x12\n\n\x02id\x18\x01 \x01(\t\x12\x1c\n\x14protocol_instance_id\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12.\n\x05input\x18\x04 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x43\n\x07headers\x18\x05 \x03(\x0b\x32\x32.coresdk.workflow_activation.DoUpdate.HeadersEntry\x12*\n\x04meta\x18\x06 \x01(\x0b\x32\x1c.temporal.api.update.v1.Meta\x12\x15\n\rrun_validator\x18\x07 \x01(\x08\x12\x65\n&propagated_nexus_serialization_context\x18\x08 \x01(\x0b\x32\x35.temporal.api.nexus.v1.PropagatedSerializationContext\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x9a\x01\n\x1aResolveNexusOperationStart\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x19\n\x0foperation_token\x18\x02 \x01(\tH\x00\x12\x16\n\x0cstarted_sync\x18\x03 \x01(\x08H\x00\x12\x32\n\x06\x66\x61iled\x18\x04 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x42\x08\n\x06status"Y\n\x15ResolveNexusOperation\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x33\n\x06result\x18\x02 \x01(\x0b\x32#.coresdk.nexus.NexusOperationResult"\xe0\x02\n\x0fRemoveFromCache\x12\x0f\n\x07message\x18\x01 \x01(\t\x12K\n\x06reason\x18\x02 \x01(\x0e\x32;.coresdk.workflow_activation.RemoveFromCache.EvictionReason"\xee\x01\n\x0e\x45victionReason\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x0e\n\nCACHE_FULL\x10\x01\x12\x0e\n\nCACHE_MISS\x10\x02\x12\x12\n\x0eNONDETERMINISM\x10\x03\x12\r\n\tLANG_FAIL\x10\x04\x12\x12\n\x0eLANG_REQUESTED\x10\x05\x12\x12\n\x0eTASK_NOT_FOUND\x10\x06\x12\x15\n\x11UNHANDLED_COMMAND\x10\x07\x12\t\n\x05\x46\x41TAL\x10\x08\x12\x1f\n\x1bPAGINATION_OR_HISTORY_FETCH\x10\t\x12\x1d\n\x19WORKFLOW_EXECUTION_ENDING\x10\n"\x90\x01\n\x11ResolveStreamRead\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x35\n\x07records\x18\x02 \x03(\x0b\x32$.temporal.api.stream.v1.StreamRecord\x12\x37\n\x03\x65nd\x18\x03 \x01(\x0b\x32*.coresdk.workflow_activation.StreamReadEnd"\x93\x01\n\rStreamReadEnd\x12\x13\n\x0blast_offset\x18\x01 \x01(\x0c\x12,\n\x04stop\x18\x02 \x01(\x0e\x32\x1e.coresdk.common.StreamReadStop\x12\x0c\n\x04\x64one\x18\x03 \x01(\x08\x12\x31\n\x07\x66\x61ilure\x18\x04 \x01(\x0b\x32 .temporal.api.failure.v1.Failure"\xa1\x01\n\x13ResolveStreamAppend\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12?\n\x08\x61ppended\x18\x02 \x01(\x0b\x32+.coresdk.workflow_activation.StreamAppendedH\x00\x12\x32\n\x06\x66\x61iled\x18\x03 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x42\x08\n\x06status";\n\x0eStreamAppended\x12\x14\n\x0c\x66irst_offset\x18\x01 \x01(\x0c\x12\x13\n\x0blast_offset\x18\x02 \x01(\x0c"S\n\x12ResolveStreamClose\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x30\n\x06\x66\x61iled\x18\x02 \x01(\x0b\x32 .temporal.api.failure.v1.FailureB8\xea\x02\x35Temporalio::Internal::Bridge::Api::WorkflowActivationb\x06proto3' ) _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) @@ -67,56 +70,66 @@ _SIGNALWORKFLOW_HEADERSENTRY._serialized_options = b"8\001" _DOUPDATE_HEADERSENTRY._options = None _DOUPDATE_HEADERSENTRY._serialized_options = b"8\001" - _WORKFLOWACTIVATION._serialized_start = 611 - _WORKFLOWACTIVATION._serialized_end = 1157 - _WORKFLOWACTIVATIONJOB._serialized_start = 1160 - _WORKFLOWACTIVATIONJOB._serialized_end = 2536 - _INITIALIZEWORKFLOW._serialized_start = 2539 - _INITIALIZEWORKFLOW._serialized_end = 4046 - _INITIALIZEWORKFLOW_HEADERSENTRY._serialized_start = 3967 - _INITIALIZEWORKFLOW_HEADERSENTRY._serialized_end = 4046 - _FIRETIMER._serialized_start = 4048 - _FIRETIMER._serialized_end = 4072 - _RESOLVEACTIVITY._serialized_start = 4074 - _RESOLVEACTIVITY._serialized_end = 4183 - _RESOLVECHILDWORKFLOWEXECUTIONSTART._serialized_start = 4186 - _RESOLVECHILDWORKFLOWEXECUTIONSTART._serialized_end = 4523 - _RESOLVECHILDWORKFLOWEXECUTIONSTARTSUCCESS._serialized_start = 4525 - _RESOLVECHILDWORKFLOWEXECUTIONSTARTSUCCESS._serialized_end = 4584 - _RESOLVECHILDWORKFLOWEXECUTIONSTARTFAILURE._serialized_start = 4587 - _RESOLVECHILDWORKFLOWEXECUTIONSTARTFAILURE._serialized_end = 4753 - _RESOLVECHILDWORKFLOWEXECUTIONSTARTCANCELLED._serialized_start = 4755 - _RESOLVECHILDWORKFLOWEXECUTIONSTARTCANCELLED._serialized_end = 4851 - _RESOLVECHILDWORKFLOWEXECUTION._serialized_start = 4853 - _RESOLVECHILDWORKFLOWEXECUTION._serialized_end = 4958 - _UPDATERANDOMSEED._serialized_start = 4960 - _UPDATERANDOMSEED._serialized_end = 5003 - _QUERYWORKFLOW._serialized_start = 5006 - _QUERYWORKFLOW._serialized_end = 5266 - _QUERYWORKFLOW_HEADERSENTRY._serialized_start = 3967 - _QUERYWORKFLOW_HEADERSENTRY._serialized_end = 4046 - _CANCELWORKFLOW._serialized_start = 5268 - _CANCELWORKFLOW._serialized_end = 5300 - _SIGNALWORKFLOW._serialized_start = 5303 - _SIGNALWORKFLOW._serialized_end = 5592 - _SIGNALWORKFLOW_HEADERSENTRY._serialized_start = 3967 - _SIGNALWORKFLOW_HEADERSENTRY._serialized_end = 4046 - _NOTIFYHASPATCH._serialized_start = 5594 - _NOTIFYHASPATCH._serialized_end = 5628 - _RESOLVESIGNALEXTERNALWORKFLOW._serialized_start = 5631 - _RESOLVESIGNALEXTERNALWORKFLOW._serialized_end = 5808 - _RESOLVEREQUESTCANCELEXTERNALWORKFLOW._serialized_start = 5811 - _RESOLVEREQUESTCANCELEXTERNALWORKFLOW._serialized_end = 5995 - _DOUPDATE._serialized_start = 5998 - _DOUPDATE._serialized_end = 6432 - _DOUPDATE_HEADERSENTRY._serialized_start = 3967 - _DOUPDATE_HEADERSENTRY._serialized_end = 4046 - _RESOLVENEXUSOPERATIONSTART._serialized_start = 6435 - _RESOLVENEXUSOPERATIONSTART._serialized_end = 6589 - _RESOLVENEXUSOPERATION._serialized_start = 6591 - _RESOLVENEXUSOPERATION._serialized_end = 6680 - _REMOVEFROMCACHE._serialized_start = 6683 - _REMOVEFROMCACHE._serialized_end = 7035 - _REMOVEFROMCACHE_EVICTIONREASON._serialized_start = 6797 - _REMOVEFROMCACHE_EVICTIONREASON._serialized_end = 7035 + _WORKFLOWACTIVATION._serialized_start = 649 + _WORKFLOWACTIVATION._serialized_end = 1195 + _WORKFLOWACTIVATIONJOB._serialized_start = 1198 + _WORKFLOWACTIVATIONJOB._serialized_end = 2817 + _INITIALIZEWORKFLOW._serialized_start = 2820 + _INITIALIZEWORKFLOW._serialized_end = 4327 + _INITIALIZEWORKFLOW_HEADERSENTRY._serialized_start = 4248 + _INITIALIZEWORKFLOW_HEADERSENTRY._serialized_end = 4327 + _FIRETIMER._serialized_start = 4329 + _FIRETIMER._serialized_end = 4353 + _RESOLVEACTIVITY._serialized_start = 4355 + _RESOLVEACTIVITY._serialized_end = 4464 + _RESOLVECHILDWORKFLOWEXECUTIONSTART._serialized_start = 4467 + _RESOLVECHILDWORKFLOWEXECUTIONSTART._serialized_end = 4804 + _RESOLVECHILDWORKFLOWEXECUTIONSTARTSUCCESS._serialized_start = 4806 + _RESOLVECHILDWORKFLOWEXECUTIONSTARTSUCCESS._serialized_end = 4865 + _RESOLVECHILDWORKFLOWEXECUTIONSTARTFAILURE._serialized_start = 4868 + _RESOLVECHILDWORKFLOWEXECUTIONSTARTFAILURE._serialized_end = 5034 + _RESOLVECHILDWORKFLOWEXECUTIONSTARTCANCELLED._serialized_start = 5036 + _RESOLVECHILDWORKFLOWEXECUTIONSTARTCANCELLED._serialized_end = 5132 + _RESOLVECHILDWORKFLOWEXECUTION._serialized_start = 5134 + _RESOLVECHILDWORKFLOWEXECUTION._serialized_end = 5239 + _UPDATERANDOMSEED._serialized_start = 5241 + _UPDATERANDOMSEED._serialized_end = 5284 + _QUERYWORKFLOW._serialized_start = 5287 + _QUERYWORKFLOW._serialized_end = 5547 + _QUERYWORKFLOW_HEADERSENTRY._serialized_start = 4248 + _QUERYWORKFLOW_HEADERSENTRY._serialized_end = 4327 + _CANCELWORKFLOW._serialized_start = 5549 + _CANCELWORKFLOW._serialized_end = 5581 + _SIGNALWORKFLOW._serialized_start = 5584 + _SIGNALWORKFLOW._serialized_end = 5873 + _SIGNALWORKFLOW_HEADERSENTRY._serialized_start = 4248 + _SIGNALWORKFLOW_HEADERSENTRY._serialized_end = 4327 + _NOTIFYHASPATCH._serialized_start = 5875 + _NOTIFYHASPATCH._serialized_end = 5909 + _RESOLVESIGNALEXTERNALWORKFLOW._serialized_start = 5912 + _RESOLVESIGNALEXTERNALWORKFLOW._serialized_end = 6089 + _RESOLVEREQUESTCANCELEXTERNALWORKFLOW._serialized_start = 6092 + _RESOLVEREQUESTCANCELEXTERNALWORKFLOW._serialized_end = 6276 + _DOUPDATE._serialized_start = 6279 + _DOUPDATE._serialized_end = 6713 + _DOUPDATE_HEADERSENTRY._serialized_start = 4248 + _DOUPDATE_HEADERSENTRY._serialized_end = 4327 + _RESOLVENEXUSOPERATIONSTART._serialized_start = 6716 + _RESOLVENEXUSOPERATIONSTART._serialized_end = 6870 + _RESOLVENEXUSOPERATION._serialized_start = 6872 + _RESOLVENEXUSOPERATION._serialized_end = 6961 + _REMOVEFROMCACHE._serialized_start = 6964 + _REMOVEFROMCACHE._serialized_end = 7316 + _REMOVEFROMCACHE_EVICTIONREASON._serialized_start = 7078 + _REMOVEFROMCACHE_EVICTIONREASON._serialized_end = 7316 + _RESOLVESTREAMREAD._serialized_start = 7319 + _RESOLVESTREAMREAD._serialized_end = 7463 + _STREAMREADEND._serialized_start = 7466 + _STREAMREADEND._serialized_end = 7613 + _RESOLVESTREAMAPPEND._serialized_start = 7616 + _RESOLVESTREAMAPPEND._serialized_end = 7777 + _STREAMAPPENDED._serialized_start = 7779 + _STREAMAPPENDED._serialized_end = 7838 + _RESOLVESTREAMCLOSE._serialized_start = 7840 + _RESOLVESTREAMCLOSE._serialized_end = 7923 # @@protoc_insertion_point(module_scope) diff --git a/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.pyi b/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.pyi index af1a2e38ad..de21dd362c 100644 --- a/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.pyi +++ b/temporalio/bridge/proto/workflow_activation/workflow_activation_pb2.pyi @@ -23,6 +23,7 @@ import temporalio.api.enums.v1.failed_cause_pb2 import temporalio.api.enums.v1.workflow_pb2 import temporalio.api.failure.v1.message_pb2 import temporalio.api.nexus.v1.message_pb2 +import temporalio.api.stream.v1.message_pb2 import temporalio.api.update.v1.message_pb2 import temporalio.bridge.proto.activity_result.activity_result_pb2 import temporalio.bridge.proto.child_workflow.child_workflow_pb2 @@ -244,6 +245,9 @@ class WorkflowActivationJob(google.protobuf.message.Message): DO_UPDATE_FIELD_NUMBER: builtins.int RESOLVE_NEXUS_OPERATION_START_FIELD_NUMBER: builtins.int RESOLVE_NEXUS_OPERATION_FIELD_NUMBER: builtins.int + RESOLVE_STREAM_READ_FIELD_NUMBER: builtins.int + RESOLVE_STREAM_APPEND_FIELD_NUMBER: builtins.int + RESOLVE_STREAM_CLOSE_FIELD_NUMBER: builtins.int REMOVE_FROM_CACHE_FIELD_NUMBER: builtins.int @property def initialize_workflow(self) -> global___InitializeWorkflow: @@ -318,6 +322,18 @@ class WorkflowActivationJob(google.protobuf.message.Message): def resolve_nexus_operation(self) -> global___ResolveNexusOperation: """A nexus operation resolved.""" + @property + def resolve_stream_read(self) -> global___ResolveStreamRead: + """A read of a Stream gave Records, ended, or both.""" + + @property + def resolve_stream_append(self) -> global___ResolveStreamAppend: + """An append to a Stream resolved.""" + + @property + def resolve_stream_close(self) -> global___ResolveStreamClose: + """A close of a Stream resolved.""" + @property def remove_from_cache(self) -> global___RemoveFromCache: """Remove the workflow identified by the [WorkflowActivation] containing this job from the @@ -347,6 +363,9 @@ class WorkflowActivationJob(google.protobuf.message.Message): do_update: global___DoUpdate | None = ..., resolve_nexus_operation_start: global___ResolveNexusOperationStart | None = ..., resolve_nexus_operation: global___ResolveNexusOperation | None = ..., + resolve_stream_read: global___ResolveStreamRead | None = ..., + resolve_stream_append: global___ResolveStreamAppend | None = ..., + resolve_stream_close: global___ResolveStreamClose | None = ..., remove_from_cache: global___RemoveFromCache | None = ..., ) -> None: ... def HasField( @@ -380,6 +399,12 @@ class WorkflowActivationJob(google.protobuf.message.Message): b"resolve_request_cancel_external_workflow", "resolve_signal_external_workflow", b"resolve_signal_external_workflow", + "resolve_stream_append", + b"resolve_stream_append", + "resolve_stream_close", + b"resolve_stream_close", + "resolve_stream_read", + b"resolve_stream_read", "signal_workflow", b"signal_workflow", "update_random_seed", @@ -419,6 +444,12 @@ class WorkflowActivationJob(google.protobuf.message.Message): b"resolve_request_cancel_external_workflow", "resolve_signal_external_workflow", b"resolve_signal_external_workflow", + "resolve_stream_append", + b"resolve_stream_append", + "resolve_stream_close", + b"resolve_stream_close", + "resolve_stream_read", + b"resolve_stream_read", "signal_workflow", b"signal_workflow", "update_random_seed", @@ -446,6 +477,9 @@ class WorkflowActivationJob(google.protobuf.message.Message): "do_update", "resolve_nexus_operation_start", "resolve_nexus_operation", + "resolve_stream_read", + "resolve_stream_append", + "resolve_stream_close", "remove_from_cache", ] | None @@ -1639,3 +1673,186 @@ class RemoveFromCache(google.protobuf.message.Message): ) -> None: ... global___RemoveFromCache = RemoveFromCache + +@typing.final +class ResolveStreamRead(google.protobuf.message.Message): + """The Records that a read of a Stream gave since its last job, and its end if it ended.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SEQ_FIELD_NUMBER: builtins.int + RECORDS_FIELD_NUMBER: builtins.int + END_FIELD_NUMBER: builtins.int + seq: builtins.int + """The sequence number of the `StartStreamRead`""" + @property + def records( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + temporalio.api.stream.v1.message_pb2.StreamRecord + ]: ... + @property + def end(self) -> global___StreamReadEnd: + """Set on the last job of a read.""" + + def __init__( + self, + *, + seq: builtins.int = ..., + records: collections.abc.Iterable[ + temporalio.api.stream.v1.message_pb2.StreamRecord + ] + | None = ..., + end: global___StreamReadEnd | None = ..., + ) -> None: ... + def HasField(self, field_name: typing.Literal["end", b"end"]) -> builtins.bool: ... + def ClearField( + self, + field_name: typing.Literal["end", b"end", "records", b"records", "seq", b"seq"], + ) -> None: ... + +global___ResolveStreamRead = ResolveStreamRead + +@typing.final +class StreamReadEnd(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + LAST_OFFSET_FIELD_NUMBER: builtins.int + STOP_FIELD_NUMBER: builtins.int + DONE_FIELD_NUMBER: builtins.int + FAILURE_FIELD_NUMBER: builtins.int + last_offset: builtins.bytes + """The offset of the last Record that the workflow got, or `from_offset` if it got none.""" + stop: temporalio.bridge.proto.common.common_pb2.StreamReadStop.ValueType + """Why the read ended. Unspecified when it failed.""" + done: builtins.bool + """True if the Stream is closed and the read gave its last Record.""" + @property + def failure(self) -> temporalio.api.failure.v1.message_pb2.Failure: + """Set if the read failed.""" + + def __init__( + self, + *, + last_offset: builtins.bytes = ..., + stop: temporalio.bridge.proto.common.common_pb2.StreamReadStop.ValueType = ..., + done: builtins.bool = ..., + failure: temporalio.api.failure.v1.message_pb2.Failure | None = ..., + ) -> None: ... + def HasField( + self, field_name: typing.Literal["failure", b"failure"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing.Literal[ + "done", + b"done", + "failure", + b"failure", + "last_offset", + b"last_offset", + "stop", + b"stop", + ], + ) -> None: ... + +global___StreamReadEnd = StreamReadEnd + +@typing.final +class ResolveStreamAppend(google.protobuf.message.Message): + """The result of an `AppendToStream`.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SEQ_FIELD_NUMBER: builtins.int + APPENDED_FIELD_NUMBER: builtins.int + FAILED_FIELD_NUMBER: builtins.int + seq: builtins.int + """The sequence number of the `AppendToStream`""" + @property + def appended(self) -> global___StreamAppended: ... + @property + def failed(self) -> temporalio.api.failure.v1.message_pb2.Failure: ... + def __init__( + self, + *, + seq: builtins.int = ..., + appended: global___StreamAppended | None = ..., + failed: temporalio.api.failure.v1.message_pb2.Failure | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing.Literal[ + "appended", b"appended", "failed", b"failed", "status", b"status" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing.Literal[ + "appended", + b"appended", + "failed", + b"failed", + "seq", + b"seq", + "status", + b"status", + ], + ) -> None: ... + def WhichOneof( + self, oneof_group: typing.Literal["status", b"status"] + ) -> typing.Literal["appended", "failed"] | None: ... + +global___ResolveStreamAppend = ResolveStreamAppend + +@typing.final +class StreamAppended(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FIRST_OFFSET_FIELD_NUMBER: builtins.int + LAST_OFFSET_FIELD_NUMBER: builtins.int + first_offset: builtins.bytes + last_offset: builtins.bytes + def __init__( + self, + *, + first_offset: builtins.bytes = ..., + last_offset: builtins.bytes = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing.Literal[ + "first_offset", b"first_offset", "last_offset", b"last_offset" + ], + ) -> None: ... + +global___StreamAppended = StreamAppended + +@typing.final +class ResolveStreamClose(google.protobuf.message.Message): + """The result of a `CloseStream`, in a workflow task after the one that asked for it.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SEQ_FIELD_NUMBER: builtins.int + FAILED_FIELD_NUMBER: builtins.int + seq: builtins.int + """The sequence number of the `CloseStream`""" + @property + def failed(self) -> temporalio.api.failure.v1.message_pb2.Failure: + """Unset when the Stream is closed, also when it was closed already.""" + + def __init__( + self, + *, + seq: builtins.int = ..., + failed: temporalio.api.failure.v1.message_pb2.Failure | None = ..., + ) -> None: ... + def HasField( + self, field_name: typing.Literal["failed", b"failed"] + ) -> builtins.bool: ... + def ClearField( + self, field_name: typing.Literal["failed", b"failed", "seq", b"seq"] + ) -> None: ... + +global___ResolveStreamClose = ResolveStreamClose diff --git a/temporalio/bridge/proto/workflow_commands/__init__.py b/temporalio/bridge/proto/workflow_commands/__init__.py index f599917afa..15dde24ee9 100644 --- a/temporalio/bridge/proto/workflow_commands/__init__.py +++ b/temporalio/bridge/proto/workflow_commands/__init__.py @@ -1,9 +1,12 @@ from .workflow_commands_pb2 import ( ActivityCancellationType, + AppendToStream, CancelChildWorkflowExecution, CancelSignalWorkflow, CancelTimer, CancelWorkflowExecution, + CloseStream, + CloseStreamRead, CompleteWorkflowExecution, ContinueAsNewWorkflowExecution, FailWorkflowExecution, @@ -20,6 +23,7 @@ SetPatchMarker, SignalExternalWorkflowExecution, StartChildWorkflowExecution, + StartStreamRead, StartTimer, UpdateResponse, UpsertWorkflowSearchAttributes, @@ -28,10 +32,13 @@ __all__ = [ "ActivityCancellationType", + "AppendToStream", "CancelChildWorkflowExecution", "CancelSignalWorkflow", "CancelTimer", "CancelWorkflowExecution", + "CloseStream", + "CloseStreamRead", "CompleteWorkflowExecution", "ContinueAsNewWorkflowExecution", "FailWorkflowExecution", @@ -48,6 +55,7 @@ "SetPatchMarker", "SignalExternalWorkflowExecution", "StartChildWorkflowExecution", + "StartStreamRead", "StartTimer", "UpdateResponse", "UpsertWorkflowSearchAttributes", diff --git a/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.py b/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.py index 3573f36b11..1908ea226e 100644 --- a/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.py +++ b/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.py @@ -32,6 +32,9 @@ from temporalio.api.sdk.v1 import ( user_metadata_pb2 as temporal_dot_api_dot_sdk_dot_v1_dot_user__metadata__pb2, ) +from temporalio.api.stream.v1 import ( + message_pb2 as temporal_dot_api_dot_stream_dot_v1_dot_message__pb2, +) from temporalio.api.workflow.v1 import ( message_pb2 as temporal_dot_api_dot_workflow_dot_v1_dot_message__pb2, ) @@ -46,7 +49,7 @@ ) DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n;temporal/sdk/core/workflow_commands/workflow_commands.proto\x12\x19\x63oresdk.workflow_commands\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a$temporal/api/common/v1/message.proto\x1a$temporal/api/enums/v1/workflow.proto\x1a%temporal/api/failure/v1/message.proto\x1a\'temporal/api/sdk/v1/user_metadata.proto\x1a,temporal/api/sdk/v1/event_group_marker.proto\x1a&temporal/api/workflow/v1/message.proto\x1a\x35temporal/sdk/core/child_workflow/child_workflow.proto\x1a#temporal/sdk/core/nexus/nexus.proto\x1a%temporal/sdk/core/common/common.proto"\xa9\x10\n\x0fWorkflowCommand\x12\x38\n\ruser_metadata\x18\x64 \x01(\x0b\x32!.temporal.api.sdk.v1.UserMetadata\x12\x42\n\x13\x65vent_group_markers\x18\x65 \x03(\x0b\x32%.temporal.api.sdk.v1.EventGroupMarker\x12<\n\x0bstart_timer\x18\x01 \x01(\x0b\x32%.coresdk.workflow_commands.StartTimerH\x00\x12H\n\x11schedule_activity\x18\x02 \x01(\x0b\x32+.coresdk.workflow_commands.ScheduleActivityH\x00\x12\x42\n\x10respond_to_query\x18\x03 \x01(\x0b\x32&.coresdk.workflow_commands.QueryResultH\x00\x12S\n\x17request_cancel_activity\x18\x04 \x01(\x0b\x32\x30.coresdk.workflow_commands.RequestCancelActivityH\x00\x12>\n\x0c\x63\x61ncel_timer\x18\x05 \x01(\x0b\x32&.coresdk.workflow_commands.CancelTimerH\x00\x12[\n\x1b\x63omplete_workflow_execution\x18\x06 \x01(\x0b\x32\x34.coresdk.workflow_commands.CompleteWorkflowExecutionH\x00\x12S\n\x17\x66\x61il_workflow_execution\x18\x07 \x01(\x0b\x32\x30.coresdk.workflow_commands.FailWorkflowExecutionH\x00\x12g\n"continue_as_new_workflow_execution\x18\x08 \x01(\x0b\x32\x39.coresdk.workflow_commands.ContinueAsNewWorkflowExecutionH\x00\x12W\n\x19\x63\x61ncel_workflow_execution\x18\t \x01(\x0b\x32\x32.coresdk.workflow_commands.CancelWorkflowExecutionH\x00\x12\x45\n\x10set_patch_marker\x18\n \x01(\x0b\x32).coresdk.workflow_commands.SetPatchMarkerH\x00\x12`\n\x1estart_child_workflow_execution\x18\x0b \x01(\x0b\x32\x36.coresdk.workflow_commands.StartChildWorkflowExecutionH\x00\x12\x62\n\x1f\x63\x61ncel_child_workflow_execution\x18\x0c \x01(\x0b\x32\x37.coresdk.workflow_commands.CancelChildWorkflowExecutionH\x00\x12w\n*request_cancel_external_workflow_execution\x18\r \x01(\x0b\x32\x41.coresdk.workflow_commands.RequestCancelExternalWorkflowExecutionH\x00\x12h\n"signal_external_workflow_execution\x18\x0e \x01(\x0b\x32:.coresdk.workflow_commands.SignalExternalWorkflowExecutionH\x00\x12Q\n\x16\x63\x61ncel_signal_workflow\x18\x0f \x01(\x0b\x32/.coresdk.workflow_commands.CancelSignalWorkflowH\x00\x12S\n\x17schedule_local_activity\x18\x10 \x01(\x0b\x32\x30.coresdk.workflow_commands.ScheduleLocalActivityH\x00\x12^\n\x1drequest_cancel_local_activity\x18\x11 \x01(\x0b\x32\x35.coresdk.workflow_commands.RequestCancelLocalActivityH\x00\x12\x66\n!upsert_workflow_search_attributes\x18\x12 \x01(\x0b\x32\x39.coresdk.workflow_commands.UpsertWorkflowSearchAttributesH\x00\x12Y\n\x1amodify_workflow_properties\x18\x13 \x01(\x0b\x32\x33.coresdk.workflow_commands.ModifyWorkflowPropertiesH\x00\x12\x44\n\x0fupdate_response\x18\x14 \x01(\x0b\x32).coresdk.workflow_commands.UpdateResponseH\x00\x12U\n\x18schedule_nexus_operation\x18\x15 \x01(\x0b\x32\x31.coresdk.workflow_commands.ScheduleNexusOperationH\x00\x12`\n\x1erequest_cancel_nexus_operation\x18\x16 \x01(\x0b\x32\x36.coresdk.workflow_commands.RequestCancelNexusOperationH\x00\x42\t\n\x07variant"S\n\nStartTimer\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x38\n\x15start_to_fire_timeout\x18\x02 \x01(\x0b\x32\x19.google.protobuf.Duration"\x1a\n\x0b\x43\x61ncelTimer\x12\x0b\n\x03seq\x18\x01 \x01(\r"\xb8\x06\n\x10ScheduleActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x13\n\x0b\x61\x63tivity_id\x18\x02 \x01(\t\x12\x15\n\ractivity_type\x18\x03 \x01(\t\x12\x12\n\ntask_queue\x18\x05 \x01(\t\x12I\n\x07headers\x18\x06 \x03(\x0b\x32\x38.coresdk.workflow_commands.ScheduleActivity.HeadersEntry\x12\x32\n\targuments\x18\x07 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12<\n\x19schedule_to_close_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12<\n\x19schedule_to_start_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x34\n\x11heartbeat_timeout\x18\x0b \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x0cretry_policy\x18\x0c \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12N\n\x11\x63\x61ncellation_type\x18\r \x01(\x0e\x32\x33.coresdk.workflow_commands.ActivityCancellationType\x12\x1e\n\x16\x64o_not_eagerly_execute\x18\x0e \x01(\x08\x12;\n\x11versioning_intent\x18\x0f \x01(\x0e\x32 .coresdk.common.VersioningIntent\x12\x32\n\x08priority\x18\x10 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x93\x06\n\x15ScheduleLocalActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x13\n\x0b\x61\x63tivity_id\x18\x02 \x01(\t\x12\x15\n\ractivity_type\x18\x03 \x01(\t\x12\x0f\n\x07\x61ttempt\x18\x04 \x01(\r\x12:\n\x16original_schedule_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12N\n\x07headers\x18\x06 \x03(\x0b\x32=.coresdk.workflow_commands.ScheduleLocalActivity.HeadersEntry\x12\x32\n\targuments\x18\x07 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12<\n\x19schedule_to_close_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12<\n\x19schedule_to_start_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x0cretry_policy\x18\x0b \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x38\n\x15local_retry_threshold\x18\x0c \x01(\x0b\x32\x19.google.protobuf.Duration\x12N\n\x11\x63\x61ncellation_type\x18\r \x01(\x0e\x32\x33.coresdk.workflow_commands.ActivityCancellationType\x12#\n\x1binclude_arguments_in_marker\x18\x0e \x01(\x08\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"$\n\x15RequestCancelActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r")\n\x1aRequestCancelLocalActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r"\x9c\x01\n\x0bQueryResult\x12\x10\n\x08query_id\x18\x01 \x01(\t\x12<\n\tsucceeded\x18\x02 \x01(\x0b\x32\'.coresdk.workflow_commands.QuerySuccessH\x00\x12\x32\n\x06\x66\x61iled\x18\x03 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x42\t\n\x07variant"A\n\x0cQuerySuccess\x12\x31\n\x08response\x18\x01 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload"L\n\x19\x43ompleteWorkflowExecution\x12/\n\x06result\x18\x01 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload"J\n\x15\x46\x61ilWorkflowExecution\x12\x31\n\x07\x66\x61ilure\x18\x01 \x01(\x0b\x32 .temporal.api.failure.v1.Failure"\x92\x07\n\x1e\x43ontinueAsNewWorkflowExecution\x12\x15\n\rworkflow_type\x18\x01 \x01(\t\x12\x12\n\ntask_queue\x18\x02 \x01(\t\x12\x32\n\targuments\x18\x03 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x37\n\x14workflow_run_timeout\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\x05 \x01(\x0b\x32\x19.google.protobuf.Duration\x12Q\n\x04memo\x18\x06 \x03(\x0b\x32\x43.coresdk.workflow_commands.ContinueAsNewWorkflowExecution.MemoEntry\x12W\n\x07headers\x18\x07 \x03(\x0b\x32\x46.coresdk.workflow_commands.ContinueAsNewWorkflowExecution.HeadersEntry\x12\x43\n\x11search_attributes\x18\x08 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12\x39\n\x0cretry_policy\x18\t \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12;\n\x11versioning_intent\x18\n \x01(\x0e\x32 .coresdk.common.VersioningIntent\x12[\n\x1binitial_versioning_behavior\x18\x0b \x01(\x0e\x32\x36.temporal.api.enums.v1.ContinueAsNewVersioningBehavior\x12\x39\n\x16\x62\x61\x63koff_start_interval\x18\x0c \x01(\x0b\x32\x19.google.protobuf.Duration\x1aL\n\tMemoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"L\n\x17\x43\x61ncelWorkflowExecution\x12\x31\n\x07\x64\x65tails\x18\x01 \x01(\x0b\x32 .temporal.api.common.v1.Payloads"6\n\x0eSetPatchMarker\x12\x10\n\x08patch_id\x18\x01 \x01(\t\x12\x12\n\ndeprecated\x18\x02 \x01(\x08"\xe1\t\n\x1bStartChildWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x03 \x01(\t\x12\x15\n\rworkflow_type\x18\x04 \x01(\t\x12\x12\n\ntask_queue\x18\x05 \x01(\t\x12.\n\x05input\x18\x06 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12=\n\x1aworkflow_execution_timeout\x18\x07 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x37\n\x14workflow_run_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x46\n\x13parent_close_policy\x18\n \x01(\x0e\x32).coresdk.child_workflow.ParentClosePolicy\x12N\n\x18workflow_id_reuse_policy\x18\x0c \x01(\x0e\x32,.temporal.api.enums.v1.WorkflowIdReusePolicy\x12\x39\n\x0cretry_policy\x18\r \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x15\n\rcron_schedule\x18\x0e \x01(\t\x12T\n\x07headers\x18\x0f \x03(\x0b\x32\x43.coresdk.workflow_commands.StartChildWorkflowExecution.HeadersEntry\x12N\n\x04memo\x18\x10 \x03(\x0b\x32@.coresdk.workflow_commands.StartChildWorkflowExecution.MemoEntry\x12\x43\n\x11search_attributes\x18\x11 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12P\n\x11\x63\x61ncellation_type\x18\x12 \x01(\x0e\x32\x35.coresdk.child_workflow.ChildWorkflowCancellationType\x12;\n\x11versioning_intent\x18\x13 \x01(\x0e\x32 .coresdk.common.VersioningIntent\x12\x32\n\x08priority\x18\x14 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12I\n\x13versioning_override\x18\x15 \x01(\x0b\x32,.temporal.api.workflow.v1.VersioningOverride\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01\x1aL\n\tMemoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"J\n\x1c\x43\x61ncelChildWorkflowExecution\x12\x1a\n\x12\x63hild_workflow_seq\x18\x01 \x01(\r\x12\x0e\n\x06reason\x18\x02 \x01(\t"\x8e\x01\n&RequestCancelExternalWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12G\n\x12workflow_execution\x18\x02 \x01(\x0b\x32+.coresdk.common.NamespacedWorkflowExecution\x12\x0e\n\x06reason\x18\x03 \x01(\t"\x8f\x03\n\x1fSignalExternalWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12I\n\x12workflow_execution\x18\x02 \x01(\x0b\x32+.coresdk.common.NamespacedWorkflowExecutionH\x00\x12\x1b\n\x11\x63hild_workflow_id\x18\x03 \x01(\tH\x00\x12\x13\n\x0bsignal_name\x18\x04 \x01(\t\x12-\n\x04\x61rgs\x18\x05 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12X\n\x07headers\x18\x06 \x03(\x0b\x32G.coresdk.workflow_commands.SignalExternalWorkflowExecution.HeadersEntry\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01\x42\x08\n\x06target"#\n\x14\x43\x61ncelSignalWorkflow\x12\x0b\n\x03seq\x18\x01 \x01(\r"e\n\x1eUpsertWorkflowSearchAttributes\x12\x43\n\x11search_attributes\x18\x01 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes"O\n\x18ModifyWorkflowProperties\x12\x33\n\rupserted_memo\x18\x01 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo"\xd2\x01\n\x0eUpdateResponse\x12\x1c\n\x14protocol_instance_id\x18\x01 \x01(\t\x12*\n\x08\x61\x63\x63\x65pted\x18\x02 \x01(\x0b\x32\x16.google.protobuf.EmptyH\x00\x12\x34\n\x08rejected\x18\x03 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x12\x34\n\tcompleted\x18\x04 \x01(\x0b\x32\x1f.temporal.api.common.v1.PayloadH\x00\x42\n\n\x08response"\x9a\x04\n\x16ScheduleNexusOperation\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x10\n\x08\x65ndpoint\x18\x02 \x01(\t\x12\x0f\n\x07service\x18\x03 \x01(\t\x12\x11\n\toperation\x18\x04 \x01(\t\x12.\n\x05input\x18\x05 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12<\n\x19schedule_to_close_timeout\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12X\n\x0cnexus_header\x18\x07 \x03(\x0b\x32\x42.coresdk.workflow_commands.ScheduleNexusOperation.NexusHeaderEntry\x12H\n\x11\x63\x61ncellation_type\x18\x08 \x01(\x0e\x32-.coresdk.nexus.NexusOperationCancellationType\x12<\n\x19schedule_to_start_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x1a\x32\n\x10NexusHeaderEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01"*\n\x1bRequestCancelNexusOperation\x12\x0b\n\x03seq\x18\x01 \x01(\r*X\n\x18\x41\x63tivityCancellationType\x12\x0e\n\nTRY_CANCEL\x10\x00\x12\x1f\n\x1bWAIT_CANCELLATION_COMPLETED\x10\x01\x12\x0b\n\x07\x41\x42\x41NDON\x10\x02\x42\x36\xea\x02\x33Temporalio::Internal::Bridge::Api::WorkflowCommandsb\x06proto3' + b'\n;temporal/sdk/core/workflow_commands/workflow_commands.proto\x12\x19\x63oresdk.workflow_commands\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a$temporal/api/common/v1/message.proto\x1a$temporal/api/enums/v1/workflow.proto\x1a%temporal/api/failure/v1/message.proto\x1a\'temporal/api/sdk/v1/user_metadata.proto\x1a,temporal/api/sdk/v1/event_group_marker.proto\x1a$temporal/api/stream/v1/message.proto\x1a&temporal/api/workflow/v1/message.proto\x1a\x35temporal/sdk/core/child_workflow/child_workflow.proto\x1a#temporal/sdk/core/nexus/nexus.proto\x1a%temporal/sdk/core/common/common.proto"\xc2\x12\n\x0fWorkflowCommand\x12\x38\n\ruser_metadata\x18\x64 \x01(\x0b\x32!.temporal.api.sdk.v1.UserMetadata\x12\x42\n\x13\x65vent_group_markers\x18\x65 \x03(\x0b\x32%.temporal.api.sdk.v1.EventGroupMarker\x12<\n\x0bstart_timer\x18\x01 \x01(\x0b\x32%.coresdk.workflow_commands.StartTimerH\x00\x12H\n\x11schedule_activity\x18\x02 \x01(\x0b\x32+.coresdk.workflow_commands.ScheduleActivityH\x00\x12\x42\n\x10respond_to_query\x18\x03 \x01(\x0b\x32&.coresdk.workflow_commands.QueryResultH\x00\x12S\n\x17request_cancel_activity\x18\x04 \x01(\x0b\x32\x30.coresdk.workflow_commands.RequestCancelActivityH\x00\x12>\n\x0c\x63\x61ncel_timer\x18\x05 \x01(\x0b\x32&.coresdk.workflow_commands.CancelTimerH\x00\x12[\n\x1b\x63omplete_workflow_execution\x18\x06 \x01(\x0b\x32\x34.coresdk.workflow_commands.CompleteWorkflowExecutionH\x00\x12S\n\x17\x66\x61il_workflow_execution\x18\x07 \x01(\x0b\x32\x30.coresdk.workflow_commands.FailWorkflowExecutionH\x00\x12g\n"continue_as_new_workflow_execution\x18\x08 \x01(\x0b\x32\x39.coresdk.workflow_commands.ContinueAsNewWorkflowExecutionH\x00\x12W\n\x19\x63\x61ncel_workflow_execution\x18\t \x01(\x0b\x32\x32.coresdk.workflow_commands.CancelWorkflowExecutionH\x00\x12\x45\n\x10set_patch_marker\x18\n \x01(\x0b\x32).coresdk.workflow_commands.SetPatchMarkerH\x00\x12`\n\x1estart_child_workflow_execution\x18\x0b \x01(\x0b\x32\x36.coresdk.workflow_commands.StartChildWorkflowExecutionH\x00\x12\x62\n\x1f\x63\x61ncel_child_workflow_execution\x18\x0c \x01(\x0b\x32\x37.coresdk.workflow_commands.CancelChildWorkflowExecutionH\x00\x12w\n*request_cancel_external_workflow_execution\x18\r \x01(\x0b\x32\x41.coresdk.workflow_commands.RequestCancelExternalWorkflowExecutionH\x00\x12h\n"signal_external_workflow_execution\x18\x0e \x01(\x0b\x32:.coresdk.workflow_commands.SignalExternalWorkflowExecutionH\x00\x12Q\n\x16\x63\x61ncel_signal_workflow\x18\x0f \x01(\x0b\x32/.coresdk.workflow_commands.CancelSignalWorkflowH\x00\x12S\n\x17schedule_local_activity\x18\x10 \x01(\x0b\x32\x30.coresdk.workflow_commands.ScheduleLocalActivityH\x00\x12^\n\x1drequest_cancel_local_activity\x18\x11 \x01(\x0b\x32\x35.coresdk.workflow_commands.RequestCancelLocalActivityH\x00\x12\x66\n!upsert_workflow_search_attributes\x18\x12 \x01(\x0b\x32\x39.coresdk.workflow_commands.UpsertWorkflowSearchAttributesH\x00\x12Y\n\x1amodify_workflow_properties\x18\x13 \x01(\x0b\x32\x33.coresdk.workflow_commands.ModifyWorkflowPropertiesH\x00\x12\x44\n\x0fupdate_response\x18\x14 \x01(\x0b\x32).coresdk.workflow_commands.UpdateResponseH\x00\x12U\n\x18schedule_nexus_operation\x18\x15 \x01(\x0b\x32\x31.coresdk.workflow_commands.ScheduleNexusOperationH\x00\x12`\n\x1erequest_cancel_nexus_operation\x18\x16 \x01(\x0b\x32\x36.coresdk.workflow_commands.RequestCancelNexusOperationH\x00\x12G\n\x11start_stream_read\x18\x17 \x01(\x0b\x32*.coresdk.workflow_commands.StartStreamReadH\x00\x12G\n\x11\x63lose_stream_read\x18\x18 \x01(\x0b\x32*.coresdk.workflow_commands.CloseStreamReadH\x00\x12\x45\n\x10\x61ppend_to_stream\x18\x19 \x01(\x0b\x32).coresdk.workflow_commands.AppendToStreamH\x00\x12>\n\x0c\x63lose_stream\x18\x1a \x01(\x0b\x32&.coresdk.workflow_commands.CloseStreamH\x00\x42\t\n\x07variant"S\n\nStartTimer\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x38\n\x15start_to_fire_timeout\x18\x02 \x01(\x0b\x32\x19.google.protobuf.Duration"\x1a\n\x0b\x43\x61ncelTimer\x12\x0b\n\x03seq\x18\x01 \x01(\r"\xb8\x06\n\x10ScheduleActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x13\n\x0b\x61\x63tivity_id\x18\x02 \x01(\t\x12\x15\n\ractivity_type\x18\x03 \x01(\t\x12\x12\n\ntask_queue\x18\x05 \x01(\t\x12I\n\x07headers\x18\x06 \x03(\x0b\x32\x38.coresdk.workflow_commands.ScheduleActivity.HeadersEntry\x12\x32\n\targuments\x18\x07 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12<\n\x19schedule_to_close_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12<\n\x19schedule_to_start_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x34\n\x11heartbeat_timeout\x18\x0b \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x0cretry_policy\x18\x0c \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12N\n\x11\x63\x61ncellation_type\x18\r \x01(\x0e\x32\x33.coresdk.workflow_commands.ActivityCancellationType\x12\x1e\n\x16\x64o_not_eagerly_execute\x18\x0e \x01(\x08\x12;\n\x11versioning_intent\x18\x0f \x01(\x0e\x32 .coresdk.common.VersioningIntent\x12\x32\n\x08priority\x18\x10 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x93\x06\n\x15ScheduleLocalActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x13\n\x0b\x61\x63tivity_id\x18\x02 \x01(\t\x12\x15\n\ractivity_type\x18\x03 \x01(\t\x12\x0f\n\x07\x61ttempt\x18\x04 \x01(\r\x12:\n\x16original_schedule_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12N\n\x07headers\x18\x06 \x03(\x0b\x32=.coresdk.workflow_commands.ScheduleLocalActivity.HeadersEntry\x12\x32\n\targuments\x18\x07 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12<\n\x19schedule_to_close_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12<\n\x19schedule_to_start_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x0cretry_policy\x18\x0b \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x38\n\x15local_retry_threshold\x18\x0c \x01(\x0b\x32\x19.google.protobuf.Duration\x12N\n\x11\x63\x61ncellation_type\x18\r \x01(\x0e\x32\x33.coresdk.workflow_commands.ActivityCancellationType\x12#\n\x1binclude_arguments_in_marker\x18\x0e \x01(\x08\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"$\n\x15RequestCancelActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r")\n\x1aRequestCancelLocalActivity\x12\x0b\n\x03seq\x18\x01 \x01(\r"\x9c\x01\n\x0bQueryResult\x12\x10\n\x08query_id\x18\x01 \x01(\t\x12<\n\tsucceeded\x18\x02 \x01(\x0b\x32\'.coresdk.workflow_commands.QuerySuccessH\x00\x12\x32\n\x06\x66\x61iled\x18\x03 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x42\t\n\x07variant"A\n\x0cQuerySuccess\x12\x31\n\x08response\x18\x01 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload"L\n\x19\x43ompleteWorkflowExecution\x12/\n\x06result\x18\x01 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload"J\n\x15\x46\x61ilWorkflowExecution\x12\x31\n\x07\x66\x61ilure\x18\x01 \x01(\x0b\x32 .temporal.api.failure.v1.Failure"\x92\x07\n\x1e\x43ontinueAsNewWorkflowExecution\x12\x15\n\rworkflow_type\x18\x01 \x01(\t\x12\x12\n\ntask_queue\x18\x02 \x01(\t\x12\x32\n\targuments\x18\x03 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x37\n\x14workflow_run_timeout\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\x05 \x01(\x0b\x32\x19.google.protobuf.Duration\x12Q\n\x04memo\x18\x06 \x03(\x0b\x32\x43.coresdk.workflow_commands.ContinueAsNewWorkflowExecution.MemoEntry\x12W\n\x07headers\x18\x07 \x03(\x0b\x32\x46.coresdk.workflow_commands.ContinueAsNewWorkflowExecution.HeadersEntry\x12\x43\n\x11search_attributes\x18\x08 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12\x39\n\x0cretry_policy\x18\t \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12;\n\x11versioning_intent\x18\n \x01(\x0e\x32 .coresdk.common.VersioningIntent\x12[\n\x1binitial_versioning_behavior\x18\x0b \x01(\x0e\x32\x36.temporal.api.enums.v1.ContinueAsNewVersioningBehavior\x12\x39\n\x16\x62\x61\x63koff_start_interval\x18\x0c \x01(\x0b\x32\x19.google.protobuf.Duration\x1aL\n\tMemoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"L\n\x17\x43\x61ncelWorkflowExecution\x12\x31\n\x07\x64\x65tails\x18\x01 \x01(\x0b\x32 .temporal.api.common.v1.Payloads"6\n\x0eSetPatchMarker\x12\x10\n\x08patch_id\x18\x01 \x01(\t\x12\x12\n\ndeprecated\x18\x02 \x01(\x08"\xe1\t\n\x1bStartChildWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x11\n\tnamespace\x18\x02 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x03 \x01(\t\x12\x15\n\rworkflow_type\x18\x04 \x01(\t\x12\x12\n\ntask_queue\x18\x05 \x01(\t\x12.\n\x05input\x18\x06 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12=\n\x1aworkflow_execution_timeout\x18\x07 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x37\n\x14workflow_run_timeout\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x38\n\x15workflow_task_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x46\n\x13parent_close_policy\x18\n \x01(\x0e\x32).coresdk.child_workflow.ParentClosePolicy\x12N\n\x18workflow_id_reuse_policy\x18\x0c \x01(\x0e\x32,.temporal.api.enums.v1.WorkflowIdReusePolicy\x12\x39\n\x0cretry_policy\x18\r \x01(\x0b\x32#.temporal.api.common.v1.RetryPolicy\x12\x15\n\rcron_schedule\x18\x0e \x01(\t\x12T\n\x07headers\x18\x0f \x03(\x0b\x32\x43.coresdk.workflow_commands.StartChildWorkflowExecution.HeadersEntry\x12N\n\x04memo\x18\x10 \x03(\x0b\x32@.coresdk.workflow_commands.StartChildWorkflowExecution.MemoEntry\x12\x43\n\x11search_attributes\x18\x11 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes\x12P\n\x11\x63\x61ncellation_type\x18\x12 \x01(\x0e\x32\x35.coresdk.child_workflow.ChildWorkflowCancellationType\x12;\n\x11versioning_intent\x18\x13 \x01(\x0e\x32 .coresdk.common.VersioningIntent\x12\x32\n\x08priority\x18\x14 \x01(\x0b\x32 .temporal.api.common.v1.Priority\x12I\n\x13versioning_override\x18\x15 \x01(\x0b\x32,.temporal.api.workflow.v1.VersioningOverride\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01\x1aL\n\tMemoEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"J\n\x1c\x43\x61ncelChildWorkflowExecution\x12\x1a\n\x12\x63hild_workflow_seq\x18\x01 \x01(\r\x12\x0e\n\x06reason\x18\x02 \x01(\t"\x8e\x01\n&RequestCancelExternalWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12G\n\x12workflow_execution\x18\x02 \x01(\x0b\x32+.coresdk.common.NamespacedWorkflowExecution\x12\x0e\n\x06reason\x18\x03 \x01(\t"\x8f\x03\n\x1fSignalExternalWorkflowExecution\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12I\n\x12workflow_execution\x18\x02 \x01(\x0b\x32+.coresdk.common.NamespacedWorkflowExecutionH\x00\x12\x1b\n\x11\x63hild_workflow_id\x18\x03 \x01(\tH\x00\x12\x13\n\x0bsignal_name\x18\x04 \x01(\t\x12-\n\x04\x61rgs\x18\x05 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12X\n\x07headers\x18\x06 \x03(\x0b\x32G.coresdk.workflow_commands.SignalExternalWorkflowExecution.HeadersEntry\x1aO\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01\x42\x08\n\x06target"#\n\x14\x43\x61ncelSignalWorkflow\x12\x0b\n\x03seq\x18\x01 \x01(\r"e\n\x1eUpsertWorkflowSearchAttributes\x12\x43\n\x11search_attributes\x18\x01 \x01(\x0b\x32(.temporal.api.common.v1.SearchAttributes"O\n\x18ModifyWorkflowProperties\x12\x33\n\rupserted_memo\x18\x01 \x01(\x0b\x32\x1c.temporal.api.common.v1.Memo"\xd2\x01\n\x0eUpdateResponse\x12\x1c\n\x14protocol_instance_id\x18\x01 \x01(\t\x12*\n\x08\x61\x63\x63\x65pted\x18\x02 \x01(\x0b\x32\x16.google.protobuf.EmptyH\x00\x12\x34\n\x08rejected\x18\x03 \x01(\x0b\x32 .temporal.api.failure.v1.FailureH\x00\x12\x34\n\tcompleted\x18\x04 \x01(\x0b\x32\x1f.temporal.api.common.v1.PayloadH\x00\x42\n\n\x08response"\x9a\x04\n\x16ScheduleNexusOperation\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x10\n\x08\x65ndpoint\x18\x02 \x01(\t\x12\x0f\n\x07service\x18\x03 \x01(\t\x12\x11\n\toperation\x18\x04 \x01(\t\x12.\n\x05input\x18\x05 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12<\n\x19schedule_to_close_timeout\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12X\n\x0cnexus_header\x18\x07 \x03(\x0b\x32\x42.coresdk.workflow_commands.ScheduleNexusOperation.NexusHeaderEntry\x12H\n\x11\x63\x61ncellation_type\x18\x08 \x01(\x0e\x32-.coresdk.nexus.NexusOperationCancellationType\x12<\n\x19schedule_to_start_timeout\x18\t \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x39\n\x16start_to_close_timeout\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x1a\x32\n\x10NexusHeaderEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01"*\n\x1bRequestCancelNexusOperation\x12\x0b\n\x03seq\x18\x01 \x01(\r"\x87\x02\n\x0fStartStreamRead\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x06stream\x18\x02 \x01(\x0b\x32!.temporal.api.stream.v1.StreamRef\x12\x13\n\x0b\x66rom_offset\x18\x03 \x01(\x0c\x12*\n\x07timeout\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x12/\n\x0cidle_timeout\x18\x05 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x13\n\x0bmax_records\x18\x06 \x01(\r\x12\x11\n\tmax_bytes\x18\x07 \x01(\x04\x12\x1a\n\x12records_in_history\x18\x08 \x01(\x08"\x1e\n\x0f\x43loseStreamRead\x12\x0b\n\x03seq\x18\x01 \x01(\r"\xc7\x01\n\x0e\x41ppendToStream\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x06stream\x18\x02 \x01(\x0b\x32!.temporal.api.stream.v1.StreamRef\x12/\n\x06\x62odies\x18\x03 \x03(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x32\n\x08producer\x18\x04 \x01(\x0b\x32 .temporal.api.stream.v1.Producer\x12\x10\n\x08sequence\x18\x05 \x01(\x03"\x81\x01\n\x0b\x43loseStream\x12\x0b\n\x03seq\x18\x01 \x01(\r\x12\x31\n\x06stream\x18\x02 \x01(\x0b\x32!.temporal.api.stream.v1.StreamRef\x12\x32\n\x08producer\x18\x03 \x01(\x0b\x32 .temporal.api.stream.v1.Producer*X\n\x18\x41\x63tivityCancellationType\x12\x0e\n\nTRY_CANCEL\x10\x00\x12\x1f\n\x1bWAIT_CANCELLATION_COMPLETED\x10\x01\x12\x0b\n\x07\x41\x42\x41NDON\x10\x02\x42\x36\xea\x02\x33Temporalio::Internal::Bridge::Api::WorkflowCommandsb\x06proto3' ) _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) @@ -72,70 +75,78 @@ _SIGNALEXTERNALWORKFLOWEXECUTION_HEADERSENTRY._serialized_options = b"8\001" _SCHEDULENEXUSOPERATION_NEXUSHEADERENTRY._options = None _SCHEDULENEXUSOPERATION_NEXUSHEADERENTRY._serialized_options = b"8\001" - _ACTIVITYCANCELLATIONTYPE._serialized_start = 8785 - _ACTIVITYCANCELLATIONTYPE._serialized_end = 8873 - _WORKFLOWCOMMAND._serialized_start = 558 - _WORKFLOWCOMMAND._serialized_end = 2647 - _STARTTIMER._serialized_start = 2649 - _STARTTIMER._serialized_end = 2732 - _CANCELTIMER._serialized_start = 2734 - _CANCELTIMER._serialized_end = 2760 - _SCHEDULEACTIVITY._serialized_start = 2763 - _SCHEDULEACTIVITY._serialized_end = 3587 - _SCHEDULEACTIVITY_HEADERSENTRY._serialized_start = 3508 - _SCHEDULEACTIVITY_HEADERSENTRY._serialized_end = 3587 - _SCHEDULELOCALACTIVITY._serialized_start = 3590 - _SCHEDULELOCALACTIVITY._serialized_end = 4377 - _SCHEDULELOCALACTIVITY_HEADERSENTRY._serialized_start = 3508 - _SCHEDULELOCALACTIVITY_HEADERSENTRY._serialized_end = 3587 - _REQUESTCANCELACTIVITY._serialized_start = 4379 - _REQUESTCANCELACTIVITY._serialized_end = 4415 - _REQUESTCANCELLOCALACTIVITY._serialized_start = 4417 - _REQUESTCANCELLOCALACTIVITY._serialized_end = 4458 - _QUERYRESULT._serialized_start = 4461 - _QUERYRESULT._serialized_end = 4617 - _QUERYSUCCESS._serialized_start = 4619 - _QUERYSUCCESS._serialized_end = 4684 - _COMPLETEWORKFLOWEXECUTION._serialized_start = 4686 - _COMPLETEWORKFLOWEXECUTION._serialized_end = 4762 - _FAILWORKFLOWEXECUTION._serialized_start = 4764 - _FAILWORKFLOWEXECUTION._serialized_end = 4838 - _CONTINUEASNEWWORKFLOWEXECUTION._serialized_start = 4841 - _CONTINUEASNEWWORKFLOWEXECUTION._serialized_end = 5755 - _CONTINUEASNEWWORKFLOWEXECUTION_MEMOENTRY._serialized_start = 5598 - _CONTINUEASNEWWORKFLOWEXECUTION_MEMOENTRY._serialized_end = 5674 - _CONTINUEASNEWWORKFLOWEXECUTION_HEADERSENTRY._serialized_start = 3508 - _CONTINUEASNEWWORKFLOWEXECUTION_HEADERSENTRY._serialized_end = 3587 - _CANCELWORKFLOWEXECUTION._serialized_start = 5757 - _CANCELWORKFLOWEXECUTION._serialized_end = 5833 - _SETPATCHMARKER._serialized_start = 5835 - _SETPATCHMARKER._serialized_end = 5889 - _STARTCHILDWORKFLOWEXECUTION._serialized_start = 5892 - _STARTCHILDWORKFLOWEXECUTION._serialized_end = 7141 - _STARTCHILDWORKFLOWEXECUTION_HEADERSENTRY._serialized_start = 3508 - _STARTCHILDWORKFLOWEXECUTION_HEADERSENTRY._serialized_end = 3587 - _STARTCHILDWORKFLOWEXECUTION_MEMOENTRY._serialized_start = 5598 - _STARTCHILDWORKFLOWEXECUTION_MEMOENTRY._serialized_end = 5674 - _CANCELCHILDWORKFLOWEXECUTION._serialized_start = 7143 - _CANCELCHILDWORKFLOWEXECUTION._serialized_end = 7217 - _REQUESTCANCELEXTERNALWORKFLOWEXECUTION._serialized_start = 7220 - _REQUESTCANCELEXTERNALWORKFLOWEXECUTION._serialized_end = 7362 - _SIGNALEXTERNALWORKFLOWEXECUTION._serialized_start = 7365 - _SIGNALEXTERNALWORKFLOWEXECUTION._serialized_end = 7764 - _SIGNALEXTERNALWORKFLOWEXECUTION_HEADERSENTRY._serialized_start = 3508 - _SIGNALEXTERNALWORKFLOWEXECUTION_HEADERSENTRY._serialized_end = 3587 - _CANCELSIGNALWORKFLOW._serialized_start = 7766 - _CANCELSIGNALWORKFLOW._serialized_end = 7801 - _UPSERTWORKFLOWSEARCHATTRIBUTES._serialized_start = 7803 - _UPSERTWORKFLOWSEARCHATTRIBUTES._serialized_end = 7904 - _MODIFYWORKFLOWPROPERTIES._serialized_start = 7906 - _MODIFYWORKFLOWPROPERTIES._serialized_end = 7985 - _UPDATERESPONSE._serialized_start = 7988 - _UPDATERESPONSE._serialized_end = 8198 - _SCHEDULENEXUSOPERATION._serialized_start = 8201 - _SCHEDULENEXUSOPERATION._serialized_end = 8739 - _SCHEDULENEXUSOPERATION_NEXUSHEADERENTRY._serialized_start = 8689 - _SCHEDULENEXUSOPERATION_NEXUSHEADERENTRY._serialized_end = 8739 - _REQUESTCANCELNEXUSOPERATION._serialized_start = 8741 - _REQUESTCANCELNEXUSOPERATION._serialized_end = 8783 + _ACTIVITYCANCELLATIONTYPE._serialized_start = 9736 + _ACTIVITYCANCELLATIONTYPE._serialized_end = 9824 + _WORKFLOWCOMMAND._serialized_start = 596 + _WORKFLOWCOMMAND._serialized_end = 2966 + _STARTTIMER._serialized_start = 2968 + _STARTTIMER._serialized_end = 3051 + _CANCELTIMER._serialized_start = 3053 + _CANCELTIMER._serialized_end = 3079 + _SCHEDULEACTIVITY._serialized_start = 3082 + _SCHEDULEACTIVITY._serialized_end = 3906 + _SCHEDULEACTIVITY_HEADERSENTRY._serialized_start = 3827 + _SCHEDULEACTIVITY_HEADERSENTRY._serialized_end = 3906 + _SCHEDULELOCALACTIVITY._serialized_start = 3909 + _SCHEDULELOCALACTIVITY._serialized_end = 4696 + _SCHEDULELOCALACTIVITY_HEADERSENTRY._serialized_start = 3827 + _SCHEDULELOCALACTIVITY_HEADERSENTRY._serialized_end = 3906 + _REQUESTCANCELACTIVITY._serialized_start = 4698 + _REQUESTCANCELACTIVITY._serialized_end = 4734 + _REQUESTCANCELLOCALACTIVITY._serialized_start = 4736 + _REQUESTCANCELLOCALACTIVITY._serialized_end = 4777 + _QUERYRESULT._serialized_start = 4780 + _QUERYRESULT._serialized_end = 4936 + _QUERYSUCCESS._serialized_start = 4938 + _QUERYSUCCESS._serialized_end = 5003 + _COMPLETEWORKFLOWEXECUTION._serialized_start = 5005 + _COMPLETEWORKFLOWEXECUTION._serialized_end = 5081 + _FAILWORKFLOWEXECUTION._serialized_start = 5083 + _FAILWORKFLOWEXECUTION._serialized_end = 5157 + _CONTINUEASNEWWORKFLOWEXECUTION._serialized_start = 5160 + _CONTINUEASNEWWORKFLOWEXECUTION._serialized_end = 6074 + _CONTINUEASNEWWORKFLOWEXECUTION_MEMOENTRY._serialized_start = 5917 + _CONTINUEASNEWWORKFLOWEXECUTION_MEMOENTRY._serialized_end = 5993 + _CONTINUEASNEWWORKFLOWEXECUTION_HEADERSENTRY._serialized_start = 3827 + _CONTINUEASNEWWORKFLOWEXECUTION_HEADERSENTRY._serialized_end = 3906 + _CANCELWORKFLOWEXECUTION._serialized_start = 6076 + _CANCELWORKFLOWEXECUTION._serialized_end = 6152 + _SETPATCHMARKER._serialized_start = 6154 + _SETPATCHMARKER._serialized_end = 6208 + _STARTCHILDWORKFLOWEXECUTION._serialized_start = 6211 + _STARTCHILDWORKFLOWEXECUTION._serialized_end = 7460 + _STARTCHILDWORKFLOWEXECUTION_HEADERSENTRY._serialized_start = 3827 + _STARTCHILDWORKFLOWEXECUTION_HEADERSENTRY._serialized_end = 3906 + _STARTCHILDWORKFLOWEXECUTION_MEMOENTRY._serialized_start = 5917 + _STARTCHILDWORKFLOWEXECUTION_MEMOENTRY._serialized_end = 5993 + _CANCELCHILDWORKFLOWEXECUTION._serialized_start = 7462 + _CANCELCHILDWORKFLOWEXECUTION._serialized_end = 7536 + _REQUESTCANCELEXTERNALWORKFLOWEXECUTION._serialized_start = 7539 + _REQUESTCANCELEXTERNALWORKFLOWEXECUTION._serialized_end = 7681 + _SIGNALEXTERNALWORKFLOWEXECUTION._serialized_start = 7684 + _SIGNALEXTERNALWORKFLOWEXECUTION._serialized_end = 8083 + _SIGNALEXTERNALWORKFLOWEXECUTION_HEADERSENTRY._serialized_start = 3827 + _SIGNALEXTERNALWORKFLOWEXECUTION_HEADERSENTRY._serialized_end = 3906 + _CANCELSIGNALWORKFLOW._serialized_start = 8085 + _CANCELSIGNALWORKFLOW._serialized_end = 8120 + _UPSERTWORKFLOWSEARCHATTRIBUTES._serialized_start = 8122 + _UPSERTWORKFLOWSEARCHATTRIBUTES._serialized_end = 8223 + _MODIFYWORKFLOWPROPERTIES._serialized_start = 8225 + _MODIFYWORKFLOWPROPERTIES._serialized_end = 8304 + _UPDATERESPONSE._serialized_start = 8307 + _UPDATERESPONSE._serialized_end = 8517 + _SCHEDULENEXUSOPERATION._serialized_start = 8520 + _SCHEDULENEXUSOPERATION._serialized_end = 9058 + _SCHEDULENEXUSOPERATION_NEXUSHEADERENTRY._serialized_start = 9008 + _SCHEDULENEXUSOPERATION_NEXUSHEADERENTRY._serialized_end = 9058 + _REQUESTCANCELNEXUSOPERATION._serialized_start = 9060 + _REQUESTCANCELNEXUSOPERATION._serialized_end = 9102 + _STARTSTREAMREAD._serialized_start = 9105 + _STARTSTREAMREAD._serialized_end = 9368 + _CLOSESTREAMREAD._serialized_start = 9370 + _CLOSESTREAMREAD._serialized_end = 9400 + _APPENDTOSTREAM._serialized_start = 9403 + _APPENDTOSTREAM._serialized_end = 9602 + _CLOSESTREAM._serialized_start = 9605 + _CLOSESTREAM._serialized_end = 9734 # @@protoc_insertion_point(module_scope) diff --git a/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.pyi b/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.pyi index 81c47c0b03..233572f0a6 100644 --- a/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.pyi +++ b/temporalio/bridge/proto/workflow_commands/workflow_commands_pb2.pyi @@ -25,6 +25,7 @@ import temporalio.api.enums.v1.workflow_pb2 import temporalio.api.failure.v1.message_pb2 import temporalio.api.sdk.v1.event_group_marker_pb2 import temporalio.api.sdk.v1.user_metadata_pb2 +import temporalio.api.stream.v1.message_pb2 import temporalio.api.workflow.v1.message_pb2 import temporalio.bridge.proto.child_workflow.child_workflow_pb2 import temporalio.bridge.proto.common.common_pb2 @@ -105,6 +106,10 @@ class WorkflowCommand(google.protobuf.message.Message): UPDATE_RESPONSE_FIELD_NUMBER: builtins.int SCHEDULE_NEXUS_OPERATION_FIELD_NUMBER: builtins.int REQUEST_CANCEL_NEXUS_OPERATION_FIELD_NUMBER: builtins.int + START_STREAM_READ_FIELD_NUMBER: builtins.int + CLOSE_STREAM_READ_FIELD_NUMBER: builtins.int + APPEND_TO_STREAM_FIELD_NUMBER: builtins.int + CLOSE_STREAM_FIELD_NUMBER: builtins.int @property def user_metadata(self) -> temporalio.api.sdk.v1.user_metadata_pb2.UserMetadata: """User metadata that may or may not be persisted into history depending on the command type. @@ -181,6 +186,14 @@ class WorkflowCommand(google.protobuf.message.Message): def request_cancel_nexus_operation( self, ) -> global___RequestCancelNexusOperation: ... + @property + def start_stream_read(self) -> global___StartStreamRead: ... + @property + def close_stream_read(self) -> global___CloseStreamRead: ... + @property + def append_to_stream(self) -> global___AppendToStream: ... + @property + def close_stream(self) -> global___CloseStream: ... def __init__( self, *, @@ -219,10 +232,16 @@ class WorkflowCommand(google.protobuf.message.Message): schedule_nexus_operation: global___ScheduleNexusOperation | None = ..., request_cancel_nexus_operation: global___RequestCancelNexusOperation | None = ..., + start_stream_read: global___StartStreamRead | None = ..., + close_stream_read: global___CloseStreamRead | None = ..., + append_to_stream: global___AppendToStream | None = ..., + close_stream: global___CloseStream | None = ..., ) -> None: ... def HasField( self, field_name: typing.Literal[ + "append_to_stream", + b"append_to_stream", "cancel_child_workflow_execution", b"cancel_child_workflow_execution", "cancel_signal_workflow", @@ -231,6 +250,10 @@ class WorkflowCommand(google.protobuf.message.Message): b"cancel_timer", "cancel_workflow_execution", b"cancel_workflow_execution", + "close_stream", + b"close_stream", + "close_stream_read", + b"close_stream_read", "complete_workflow_execution", b"complete_workflow_execution", "continue_as_new_workflow_execution", @@ -261,6 +284,8 @@ class WorkflowCommand(google.protobuf.message.Message): b"signal_external_workflow_execution", "start_child_workflow_execution", b"start_child_workflow_execution", + "start_stream_read", + b"start_stream_read", "start_timer", b"start_timer", "update_response", @@ -276,6 +301,8 @@ class WorkflowCommand(google.protobuf.message.Message): def ClearField( self, field_name: typing.Literal[ + "append_to_stream", + b"append_to_stream", "cancel_child_workflow_execution", b"cancel_child_workflow_execution", "cancel_signal_workflow", @@ -284,6 +311,10 @@ class WorkflowCommand(google.protobuf.message.Message): b"cancel_timer", "cancel_workflow_execution", b"cancel_workflow_execution", + "close_stream", + b"close_stream", + "close_stream_read", + b"close_stream_read", "complete_workflow_execution", b"complete_workflow_execution", "continue_as_new_workflow_execution", @@ -316,6 +347,8 @@ class WorkflowCommand(google.protobuf.message.Message): b"signal_external_workflow_execution", "start_child_workflow_execution", b"start_child_workflow_execution", + "start_stream_read", + b"start_stream_read", "start_timer", b"start_timer", "update_response", @@ -354,6 +387,10 @@ class WorkflowCommand(google.protobuf.message.Message): "update_response", "schedule_nexus_operation", "request_cancel_nexus_operation", + "start_stream_read", + "close_stream_read", + "append_to_stream", + "close_stream", ] | None ): ... @@ -1931,3 +1968,211 @@ class RequestCancelNexusOperation(google.protobuf.message.Message): def ClearField(self, field_name: typing.Literal["seq", b"seq"]) -> None: ... global___RequestCancelNexusOperation = RequestCancelNexusOperation + +@typing.final +class StartStreamRead(google.protobuf.message.Message): + """Start a read of a Stream. Core reads the Store itself and gives each page of Records to the + workflow in a `ResolveStreamRead` job, until the read ends or the workflow closes it. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SEQ_FIELD_NUMBER: builtins.int + STREAM_FIELD_NUMBER: builtins.int + FROM_OFFSET_FIELD_NUMBER: builtins.int + TIMEOUT_FIELD_NUMBER: builtins.int + IDLE_TIMEOUT_FIELD_NUMBER: builtins.int + MAX_RECORDS_FIELD_NUMBER: builtins.int + MAX_BYTES_FIELD_NUMBER: builtins.int + RECORDS_IN_HISTORY_FIELD_NUMBER: builtins.int + seq: builtins.int + """Lang's incremental sequence number of Stream reads, used as the read's identifier""" + from_offset: builtins.bytes + """The read starts after this offset. Empty means the start of the Stream.""" + max_records: builtins.int + """The most Records the read gives. Zero means no cap.""" + max_bytes: builtins.int + """The most bytes of Records the read gives. The first Record is always given. Zero means no + cap. + """ + records_in_history: builtins.bool + """If set, the Records go into the workflow history, so a replay reads no Store.""" + @property + def stream(self) -> temporalio.api.stream.v1.message_pb2.StreamRef: ... + @property + def timeout(self) -> google.protobuf.duration_pb2.Duration: + """The longest time the read may take. Unset means no bound. A set value of zero or less fails + the workflow task. A read that gave no Record yet looks at the Store once before its timeout + ends it. + """ + + @property + def idle_timeout(self) -> google.protobuf.duration_pb2.Duration: + """The read ends when no Record comes for this long. Unset means 1 minute. Zero gives the + Records that are in the Stream, and ends with no wait. A read that must wait longer sets a + longer value. + """ + + def __init__( + self, + *, + seq: builtins.int = ..., + stream: temporalio.api.stream.v1.message_pb2.StreamRef | None = ..., + from_offset: builtins.bytes = ..., + timeout: google.protobuf.duration_pb2.Duration | None = ..., + idle_timeout: google.protobuf.duration_pb2.Duration | None = ..., + max_records: builtins.int = ..., + max_bytes: builtins.int = ..., + records_in_history: builtins.bool = ..., + ) -> None: ... + def HasField( + self, + field_name: typing.Literal[ + "idle_timeout", b"idle_timeout", "stream", b"stream", "timeout", b"timeout" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing.Literal[ + "from_offset", + b"from_offset", + "idle_timeout", + b"idle_timeout", + "max_bytes", + b"max_bytes", + "max_records", + b"max_records", + "records_in_history", + b"records_in_history", + "seq", + b"seq", + "stream", + b"stream", + "timeout", + b"timeout", + ], + ) -> None: ... + +global___StartStreamRead = StartStreamRead + +@typing.final +class CloseStreamRead(google.protobuf.message.Message): + """Stop a read that `StartStreamRead` started. The workflow gets no job for it.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SEQ_FIELD_NUMBER: builtins.int + seq: builtins.int + """Lang's sequence number as passed to `StartStreamRead`""" + def __init__( + self, + *, + seq: builtins.int = ..., + ) -> None: ... + def ClearField(self, field_name: typing.Literal["seq", b"seq"]) -> None: ... + +global___CloseStreamRead = CloseStreamRead + +@typing.final +class AppendToStream(google.protobuf.message.Message): + """Append Records to a Stream. Core calls the Store itself and gives the result in a + `ResolveStreamAppend` job. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SEQ_FIELD_NUMBER: builtins.int + STREAM_FIELD_NUMBER: builtins.int + BODIES_FIELD_NUMBER: builtins.int + PRODUCER_FIELD_NUMBER: builtins.int + SEQUENCE_FIELD_NUMBER: builtins.int + seq: builtins.int + """Lang's incremental sequence number of Stream appends, used as the append's identifier""" + sequence: builtins.int + """The `sequence` of the Store request, as the caller gives it. Zero means Core uses `seq`.""" + @property + def stream(self) -> temporalio.api.stream.v1.message_pb2.StreamRef: ... + @property + def bodies( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + temporalio.api.common.v1.message_pb2.Payload + ]: + """The bodies of the Records, in order""" + + @property + def producer(self) -> temporalio.api.stream.v1.message_pb2.Producer: + """The workflow's Producer, of kind workflow, with its id and run id and no epoch. Core sets the + epoch to the current workflow task. + """ + + def __init__( + self, + *, + seq: builtins.int = ..., + stream: temporalio.api.stream.v1.message_pb2.StreamRef | None = ..., + bodies: collections.abc.Iterable[temporalio.api.common.v1.message_pb2.Payload] + | None = ..., + producer: temporalio.api.stream.v1.message_pb2.Producer | None = ..., + sequence: builtins.int = ..., + ) -> None: ... + def HasField( + self, field_name: typing.Literal["producer", b"producer", "stream", b"stream"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing.Literal[ + "bodies", + b"bodies", + "producer", + b"producer", + "seq", + b"seq", + "sequence", + b"sequence", + "stream", + b"stream", + ], + ) -> None: ... + +global___AppendToStream = AppendToStream + +@typing.final +class CloseStream(google.protobuf.message.Message): + """Close a Stream. Core records the intent in this workflow task, closes the Stream after the + server took the task, and gives the result in a `ResolveStreamClose` job of a later task. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SEQ_FIELD_NUMBER: builtins.int + STREAM_FIELD_NUMBER: builtins.int + PRODUCER_FIELD_NUMBER: builtins.int + seq: builtins.int + """Lang's incremental sequence number of Stream closes, used as the close's identifier""" + @property + def stream(self) -> temporalio.api.stream.v1.message_pb2.StreamRef: ... + @property + def producer(self) -> temporalio.api.stream.v1.message_pb2.Producer: + """The workflow's Producer, of kind workflow, with its id and run id and no epoch. Core sets the + epoch to the workflow task that recorded the close. + """ + + def __init__( + self, + *, + seq: builtins.int = ..., + stream: temporalio.api.stream.v1.message_pb2.StreamRef | None = ..., + producer: temporalio.api.stream.v1.message_pb2.Producer | None = ..., + ) -> None: ... + def HasField( + self, field_name: typing.Literal["producer", b"producer", "stream", b"stream"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing.Literal[ + "producer", b"producer", "seq", b"seq", "stream", b"stream" + ], + ) -> None: ... + +global___CloseStream = CloseStream diff --git a/temporalio/bridge/sdk-core b/temporalio/bridge/sdk-core index 7bb86122ba..5d84833a12 160000 --- a/temporalio/bridge/sdk-core +++ b/temporalio/bridge/sdk-core @@ -1 +1 @@ -Subproject commit 7bb86122bab4ea1efca2295da75fa7751aeaed0c +Subproject commit 5d84833a12a13cd74866331f32b8d3d16ef2bb00 diff --git a/temporalio/bridge/src/lib.rs b/temporalio/bridge/src/lib.rs index a4fda26db6..d12d2e769b 100644 --- a/temporalio/bridge/src/lib.rs +++ b/temporalio/bridge/src/lib.rs @@ -6,6 +6,7 @@ mod client_rpc_generated; mod envconfig; mod metric; mod runtime; +mod streams; mod testing; mod worker; @@ -48,6 +49,10 @@ fn temporal_sdk_bridge(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(init_runtime, m)?)?; m.add_function(wrap_pyfunction!(raise_in_thread, m)?)?; + // Stream stuff + m.add_class::()?; + m.add_function(wrap_pyfunction!(new_python_stream_runtime, m)?)?; + // Testing stuff m.add_class::()?; m.add_function(wrap_pyfunction!(start_dev_server, m)?)?; @@ -104,6 +109,23 @@ fn raise_in_thread(py: Python, thread_id: std::os::raw::c_long, exc: &Bound<'_, runtime::raise_in_thread(py, thread_id, exc) } +#[pyfunction] +fn new_python_stream_runtime( + runtime_ref: &runtime::RuntimeRef, + caller: Py, + max_record_bytes: usize, + max_batch_bytes: usize, + max_batch_records: usize, +) -> PyResult { + streams::new_python_stream_runtime( + runtime_ref, + caller, + max_record_bytes, + max_batch_bytes, + max_batch_records, + ) +} + #[pyfunction] fn start_dev_server<'a>( py: Python<'a>, diff --git a/temporalio/bridge/src/streams.rs b/temporalio/bridge/src/streams.rs new file mode 100644 index 0000000000..c272633017 --- /dev/null +++ b/temporalio/bridge/src/streams.rs @@ -0,0 +1,201 @@ +//! The adapter that lets Core call a Stream Store written in Python for Workflow code. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; +use temporalio_streams::record::BatchLimits; + +use crate::runtime; + +/// The crate's runtime object. Each Store written in Python has one. +#[pyclass(frozen)] +pub struct StreamRuntimeRef { + pub(crate) streams: std::sync::Arc, +} + +/// Builds the runtime on a Store written in Python. Core calls the Store through it for Workflow +/// code, and the runtime checks each request and answer. `caller` runs each call of the Store on +/// the Worker's event loop, never on a Workflow thread. The three numbers are the batch limits of +/// the Store, and the runtime refuses a batch over them before the Store sees it. +pub fn new_python_stream_runtime( + runtime_ref: &runtime::RuntimeRef, + caller: Py, + max_record_bytes: usize, + max_batch_bytes: usize, + max_batch_records: usize, +) -> PyResult { + let runtime = &runtime_ref.runtime; + runtime.assert_same_process("create a Stream runtime")?; + let limits = BatchLimits::default() + .with_max_record_bytes(max_record_bytes) + .and_then(|limits| limits.with_max_batch_bytes(max_batch_bytes)) + .and_then(|limits| limits.with_max_batch_records(max_batch_records)) + .map_err(|err| PyValueError::new_err(err.message))?; + let meter = runtime.core.telemetry().get_temporal_metric_meter(); + let store = std::sync::Arc::new(python_store::PythonStore { caller, limits }); + let streams = temporalio_streams::StreamRuntime::new(store, meter); + Ok(StreamRuntimeRef { streams }) +} + +mod python_store { + use futures::channel::oneshot; + use prost::Message; + use pyo3::prelude::*; + use pyo3::types::PyBytes; + use std::sync::Mutex; + use std::time::Duration; + use temporalio_streams::{ + proto::{ + AppendRecordsRequest, AppendRecordsResponse, CloseStreamRequest, CloseStreamResponse, + CreateStreamRequest, CreateStreamResponse, DescribeStreamRequest, + DescribeStreamResponse, ReadRecordsRequest, ReadRecordsResponse, + }, + record::BatchLimits, + Reason, StreamError, StreamResult, StreamStore, + }; + + /// The longest pause after an empty page that came before its wait ended. A Store that + /// ignores `wait` then gets a read about ten times a second, and not in a busy loop. + const EMPTY_PAGE_PAUSE: Duration = Duration::from_millis(100); + + /// The encoded response, or the reason and the message of a failure. + type Answer = Result, (i32, String)>; + + /// A Store written in Python. Each call runs as a task on the Worker's event loop. + pub(crate) struct PythonStore { + /// The Python object that starts a call on the event loop and gives a canceller. + pub(crate) caller: Py, + /// The batch limits of the Store, which the runtime reads once. + pub(crate) limits: BatchLimits, + } + + /// What Python calls with the answer of one call. + #[pyclass(frozen)] + struct StoreCallDone { + tx: Mutex>>, + } + + #[pymethods] + impl StoreCallDone { + fn __call__(&self, response: Option<&[u8]>, reason: i32, message: String) { + let tx = self.tx.lock().ok().and_then(|mut tx| tx.take()); + if let Some(tx) = tx { + // Core may have stopped the call, and then nobody waits for its answer. + let _ = tx.send(match response { + Some(bytes) => Ok(bytes.to_vec()), + None => Err((reason, message)), + }); + } + } + } + + /// Cancels the Python task of a call that Core stopped, so that no task stays on the loop. + struct CancelOnDrop(Option>); + + impl Drop for CancelOnDrop { + fn drop(&mut self) { + if let Some(cancel) = self.0.take() { + // An interpreter that is shutting down has no task left to cancel. + let _ = Python::try_attach(|py| { + if let Err(err) = cancel.call0(py) { + tracing::warn!("Failed to cancel a call to a Stream Store: {err}"); + } + }); + } + } + } + + impl PythonStore { + async fn call( + &self, + operation: &'static str, + request: Req, + ) -> StreamResult { + let (tx, rx) = oneshot::channel(); + let done = StoreCallDone { + tx: Mutex::new(Some(tx)), + }; + let cancel = Python::attach(|py| { + let request = PyBytes::new(py, &request.encode_to_vec()); + self.caller + .call_method1(py, "start", (operation, request, done)) + }) + .map_err(|err| { + StreamError::storage_unavailable(format!( + "The Worker could not call the Stream Store: {err}" + )) + })?; + let mut canceller = CancelOnDrop(Some(cancel)); + let answer = rx.await; + // The task ended, so there is nothing to cancel. + canceller.0 = None; + match answer { + Ok(Ok(bytes)) => Resp::decode(bytes.as_slice()).map_err(|err| { + StreamError::storage_corrupt(format!( + "The Stream Store gave a bad {operation} response: {err}" + )) + }), + Ok(Err((value, message))) => match Reason::try_from(value) { + Ok(reason) => Err(StreamError::new(reason, message)), + Err(_) => Err(StreamError::storage_corrupt(format!( + "The Stream Store failed with the unknown reason {value}: {message}" + ))), + }, + Err(_) => Err(StreamError::storage_unavailable( + "The event loop of the Worker stopped the call to the Stream Store", + )), + } + } + } + + #[async_trait::async_trait] + impl StreamStore for PythonStore { + async fn create_stream( + &self, + request: CreateStreamRequest, + ) -> StreamResult { + self.call("CreateStream", request).await + } + + async fn append_records( + &self, + request: AppendRecordsRequest, + ) -> StreamResult { + self.call("AppendRecords", request).await + } + + async fn read_records( + &self, + request: ReadRecordsRequest, + ) -> StreamResult { + let wait = request + .wait + .and_then(|wait| Duration::try_from(wait).ok()) + .unwrap_or_default(); + let started = tokio::time::Instant::now(); + let page: ReadRecordsResponse = self.call("ReadRecords", request).await?; + let left = wait.saturating_sub(started.elapsed()); + if page.records.is_empty() && !page.done && !left.is_zero() { + tokio::time::sleep(left.min(EMPTY_PAGE_PAUSE)).await; + } + Ok(page) + } + + async fn close_stream( + &self, + request: CloseStreamRequest, + ) -> StreamResult { + self.call("CloseStream", request).await + } + + async fn describe_stream( + &self, + request: DescribeStreamRequest, + ) -> StreamResult { + self.call("DescribeStream", request).await + } + + fn limits(&self) -> BatchLimits { + self.limits + } + } +} diff --git a/temporalio/bridge/src/worker.rs b/temporalio/bridge/src/worker.rs index 39cdc5e773..c4f8848d99 100644 --- a/temporalio/bridge/src/worker.rs +++ b/temporalio/bridge/src/worker.rs @@ -5,7 +5,7 @@ use prost::Message; use pyo3::exceptions::{PyException, PyRuntimeError, PyValueError}; use pyo3::prelude::*; use pyo3::types::{PyBytes, PyTuple}; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::marker::PhantomData; use std::sync::{Arc, OnceLock}; use std::time::Duration; @@ -65,6 +65,20 @@ pub struct WorkerConfig { plugins: Vec<(String, String)>, storage_drivers: HashSet, disable_payload_error_limit: bool, + /// The Stream runtime of each Store, by Store name. Core reads and appends a Stream of + /// Workflow code through them. + stream_runtimes: Vec<(String, Py)>, + /// The most bytes of one read's Records that wait for Workflow code. `None` keeps Core's + /// default. + stream_read_ahead_bytes: Option, + /// The most Records of one read in one activation. `None` keeps Core's default. + stream_read_page_records: Option, + /// The least time between two batches of one read. `None` keeps Core's default of no gap. + stream_read_min_gap: Option, + /// Whether a replay that lost the Records of a read fails the Workflow, for every type or + /// for the types listed. + stream_replay_as_workflow_fail: bool, + stream_replay_as_workflow_fail_for_types: HashSet, } #[derive(FromPyObject)] @@ -718,6 +732,44 @@ fn convert_worker_config( ) -> PyResult { let converted_tuner = convert_tuner_holder(conf.tuner, task_locals)?; let converted_versioning_strategy = convert_versioning_strategy(conf.versioning_strategy); + let stream_runtimes = temporalio_sdk_core::StreamRuntimes::new( + conf.stream_runtimes + .into_iter() + .map(|(store, runtime)| (store, runtime.get().streams.clone())), + ); + let stream_runtimes = match conf.stream_read_ahead_bytes { + Some(bytes) => stream_runtimes.with_read_ahead_bytes(bytes), + None => stream_runtimes, + }; + let stream_runtimes = match conf.stream_read_page_records { + Some(records) => stream_runtimes.with_read_page_records(records), + None => stream_runtimes, + }; + let stream_runtimes = match conf.stream_read_min_gap { + Some(gap) => stream_runtimes.with_read_min_gap(gap), + None => stream_runtimes, + }; + let mut workflow_failure_errors = HashSet::new(); + if conf.nondeterminism_as_workflow_fail { + workflow_failure_errors.insert(WorkflowErrorType::Nondeterminism); + } + if conf.stream_replay_as_workflow_fail { + workflow_failure_errors.insert(WorkflowErrorType::StreamReplay); + } + let mut workflow_types_to_failure_errors: HashMap> = + HashMap::new(); + for workflow_type in conf.nondeterminism_as_workflow_fail_for_types { + workflow_types_to_failure_errors + .entry(workflow_type) + .or_default() + .insert(WorkflowErrorType::Nondeterminism); + } + for workflow_type in conf.stream_replay_as_workflow_fail_for_types { + workflow_types_to_failure_errors + .entry(workflow_type) + .or_default() + .insert(WorkflowErrorType::StreamReplay); + } temporalio_sdk_core::WorkerConfig::builder() .namespace(conf.namespace) .task_queue(conf.task_queue) @@ -747,22 +799,8 @@ fn convert_worker_config( // auto-cancel-activity behavior of shutdown will not occur, so we // always set it even if 0. .graceful_shutdown_period(Duration::from_millis(conf.graceful_shutdown_period_millis)) - .workflow_failure_errors(if conf.nondeterminism_as_workflow_fail { - HashSet::from([WorkflowErrorType::Nondeterminism]) - } else { - HashSet::new() - }) - .workflow_types_to_failure_errors( - conf.nondeterminism_as_workflow_fail_for_types - .iter() - .map(|s| { - ( - s.to_owned(), - HashSet::from([WorkflowErrorType::Nondeterminism]), - ) - }) - .collect(), - ) + .workflow_failure_errors(workflow_failure_errors) + .workflow_types_to_failure_errors(workflow_types_to_failure_errors) .maybe_nexus_task_poller_behavior(conf.nexus_task_poller_behavior.map(Into::into)) .plugins( conf.plugins @@ -777,6 +815,7 @@ fn convert_worker_config( .collect::>(), ) .disable_payload_error_limit(conf.disable_payload_error_limit) + .stream_runtimes(stream_runtimes) .build() .map_err(|err| PyValueError::new_err(format!("Invalid worker config: {err}"))) } diff --git a/temporalio/bridge/streams.py b/temporalio/bridge/streams.py new file mode 100644 index 0000000000..ca6a4aa204 --- /dev/null +++ b/temporalio/bridge/streams.py @@ -0,0 +1,42 @@ +"""The Stream runtime that Core uses to call a Stream Store written in Python. (unstable) + +Nothing in this module should be considered stable. The API may change. +""" + +from __future__ import annotations + +from typing import Any + +import temporalio.bridge.runtime +import temporalio.bridge.temporal_sdk_bridge +import temporalio.client + + +class StreamRuntime: + """The runtime of the ``temporalio-streams`` crate on one Store written in Python.""" + + @staticmethod + def new_python( + runtime: temporalio.bridge.runtime.Runtime, + *, + caller: Any, + limits: temporalio.client.StreamBatchLimits, + ) -> StreamRuntime: + """Build the runtime on a Store written in Python, which Core calls for Workflow code. + + ``caller`` runs each call of the Store on the Worker's event loop. Core refuses a + batch over ``limits`` before the Store sees it. + """ + return StreamRuntime( + temporalio.bridge.temporal_sdk_bridge.new_python_stream_runtime( + runtime._ref, + caller, + limits.max_record_bytes, + limits.max_batch_bytes, + limits.max_batch_records, + ) + ) + + def __init__(self, ref: Any) -> None: + """Wrap the bridge's runtime object. Use ``new_python`` instead.""" + self._ref = ref diff --git a/temporalio/bridge/worker.py b/temporalio/bridge/worker.py index fa5f337ca7..5347487a6f 100644 --- a/temporalio/bridge/worker.py +++ b/temporalio/bridge/worker.py @@ -5,9 +5,12 @@ from __future__ import annotations +import dataclasses from collections.abc import Awaitable, Callable, Sequence from dataclasses import dataclass +from datetime import timedelta from typing import ( + Any, TypeAlias, ) @@ -60,6 +63,20 @@ class WorkerConfig: plugins: Sequence[tuple[str, str]] storage_drivers: set[str] disable_payload_error_limit: bool + stream_runtimes: Sequence[tuple[str, Any]] = () + """The bridge's object of each Store's Stream runtime, by Store name.""" + stream_read_ahead_bytes: int | None = None + """The most bytes of one read's Records that wait for Workflow code, or Core's default.""" + stream_read_page_records: int | None = None + """The most Records of one read in one activation, or Core's default.""" + stream_read_min_gap: timedelta | None = None + """The least time between two batches of one read, or Core's default of no gap.""" + stream_replay_as_workflow_fail: bool = False + """Whether a replay that lost the Records of a read fails the Workflow.""" + stream_replay_as_workflow_fail_for_types: set[str] = dataclasses.field( + default_factory=set + ) + """The Workflow types for which such a replay fails the Workflow.""" @dataclass @@ -368,6 +385,7 @@ async def encode_completion( skip_search_attributes=True, skip_headers=not encode_headers, concurrency_limit=storage_concurrency_limit, + skip_stream_bodies=True, ).visit( _Visitor(data_converter._external_store_payload_sequence), completion, diff --git a/temporalio/client/__init__.py b/temporalio/client/__init__.py index 586b3b96cd..8d9870e51c 100644 --- a/temporalio/client/__init__.py +++ b/temporalio/client/__init__.py @@ -188,8 +188,10 @@ from ._stream_store import ( StreamBatchLimits, StreamConfig, + StreamReplaySource, StreamSelectContext, StreamStore, + StreamStoreCallOptions, ) from ._worker_versioning import ( BuildIdOp, @@ -358,8 +360,10 @@ "Plugin", "StreamBatchLimits", "StreamConfig", + "StreamReplaySource", "StreamSelectContext", "StreamStore", + "StreamStoreCallOptions", "Callback", "_ClientImpl", "_apply_headers", diff --git a/temporalio/client/_stream_store.py b/temporalio/client/_stream_store.py index 6b6d5778e9..d7db546af1 100644 --- a/temporalio/client/_stream_store.py +++ b/temporalio/client/_stream_store.py @@ -1,8 +1,10 @@ from __future__ import annotations import dataclasses +import enum from abc import ABC, abstractmethod -from collections.abc import Callable, Sequence +from collections.abc import Awaitable, Callable, Sequence +from datetime import timedelta from typing import Any import google.protobuf.message @@ -11,9 +13,11 @@ import temporalio.activity import temporalio.api.streamservice.v1 import temporalio.bridge._visitor +import temporalio.common import temporalio.converter import temporalio.nexus.system.stream_service -from temporalio.converter import StorageDriverActivityInfo +import temporalio.workflow +from temporalio.converter import StorageDriverActivityInfo, StorageDriverWorkflowInfo from temporalio.exceptions import StreamConfigError _MAX_RECORD_BYTES = 1536 * 1024 @@ -58,6 +62,78 @@ def __post_init__(self) -> None: raise ValueError(f"{name} must be from 1 to {most}, not {value}") +class StreamReplaySource(enum.Enum): + """Where a replay of a read of Workflow code gets the Records of the read. + + :py:attr:`StreamConfig.workflow_read_replay_from` holds it. It decides what a new read + records in History. A replay follows what History holds, so a change of the setting + doesn't break a Workflow that is running. Client and Activity code have no History, so + the setting doesn't touch them. + + .. warning:: + This API is experimental and subject to change. + """ + + HISTORY = "history" + """The marker in History holds the Records of the read with their bodies. + + A replay takes the Records from History, so it reads no Store and needs no Stream + config. Each Record adds to History. The marker of one Workflow Task holds at most + 1 MiB of Records, and a read goes on in the next task after that. + """ + + STORE = "store" + """The marker in History holds no Record and no body. This is the default. + + For each Workflow Task of a read, the marker holds the range of offsets, the count + and a digest of the Records that the Workflow got. A replay reads the Records from the + Store again and checks the count and the digest. So the replay needs the Stream config + and the Records. The Records must stay in the Store, and Retention must keep them, for + as long as the Workflow can replay. When they can't be read, the Workflow Task fails + with a message that starts with ``StreamReplayError``. A Store that stays unavailable + for 5 s fails the task with a message that starts with ``StreamReplayUnavailable`` + in its place. A replay on a Worker with no cache reads the Store again on every + Workflow Task. + """ + + +@dataclasses.dataclass(frozen=True) +class StreamStoreCallOptions: + """Options of the Local Activity that creates or describes a Stream for Workflow code. + + Workflow code can't do I/O, so the default + :py:meth:`StreamStore.call_from_workflow` calls its Store from a Local Activity. + Core makes a read, an append and a close of Workflow code itself, with no Activity, + so these options don't apply to them. Core tries such a call again while its Store + is unavailable, for at most 30 s. + + A :py:class:`StreamConfig` holds them in + :py:attr:`StreamConfig.workflow_call_options`, and every Store of that config + gets the same options. + + .. warning:: + This API is experimental and subject to change. + """ + + start_to_close_timeout: timedelta | None = timedelta(seconds=60) + """Most time for one attempt.""" + + schedule_to_close_timeout: timedelta | None = None + """Most time for all attempts.""" + + retry_policy: temporalio.common.RetryPolicy = dataclasses.field( + default_factory=lambda: temporalio.common.RetryPolicy(maximum_attempts=1) + ) + """Retries of a failed call. The default is one attempt.""" + + def __post_init__(self) -> None: + """Check that a timeout is set.""" + if not self.start_to_close_timeout and not self.schedule_to_close_timeout: + raise ValueError( + "start_to_close_timeout or schedule_to_close_timeout must be set" + ) + + class StreamStore(ABC): """A Store that answers the Stream operations of a client. @@ -66,12 +142,45 @@ class StreamStore(ABC): Store of a package, such as ``RedisStreamStore`` of ``temporalio-streams-redis``, or ``MemoryStreamStore`` of ``temporalio-streams-memory`` in tests. - To keep Streams in another system, write a Store that implements :py:attr:`name` - and the five operations. Each operation takes and gives a message of - ``temporalio.api.streamservice.v1``. The bodies of the Records arrive as payloads - that the SDK already encoded with the data converter and the payload codec, so a - Store keeps payloads and not the user's types. A failed call raises - :py:class:`temporalio.exceptions.StreamError` with a reason. + A Store is a subclass, written in Python, that implements :py:attr:`name` and the + five operations. It can call its own native code, as the Redis Store does. Each + operation takes and gives a message of ``temporalio.api.streamservice.v1``. The + bodies of the Records arrive as payloads that the SDK already encoded with the data + converter and the payload codec, so a Store keeps payloads and not the user's types. + A failed call raises :py:class:`temporalio.exceptions.StreamError` with a reason. + + Such a Store works for client, Activity and Workflow code. Core makes a read, an + append and a close of Workflow code. It calls the Store's method as a task on the + Worker's event loop, never on a Workflow thread, and records the result in History. + A create and a describe of Workflow code run in the Store's Local Activity. For + Workflow code, a Store must do these things. + + * Each method is ``async`` and never blocks the event loop, since the calls of every + Workflow on the Worker run there. + * :py:meth:`read_records` waits up to the request's ``wait`` when it has no Record + after ``from_offset``, and gives an empty page after the wait. An append or a + close wakes it. Core cancels the task of a read that it stops, for example when + Workflow code is cancelled or the Worker shuts down, so the wait must end on a + cancel. + * Each Record has an offset of 1 to 1024 bytes. Offsets of one Stream compare in + order as bytes. A page that gives the last Record of a closed Stream is ``done``. + * :py:meth:`append_records` gives the offsets of the first and the last Record. A + Workflow appends as a Producer of kind Workflow, with its Workflow Task as the + epoch. The Store writes a batch that comes again with the same Producer, epoch and + ``sequence`` once, since Core sends it again after an outage. A Store can refuse a + batch of a lower epoch of that Producer with the reason + ``STREAM_ERROR_REASON_FENCED``. It doesn't have to. + * :py:meth:`close_stream` of a closed Stream succeeds, since Core may send a close + again. + * A replay of a read under :py:attr:`StreamReplaySource.STORE` reads the Records + again, so the Store keeps them with the same offsets. + * An outage raises :py:class:`temporalio.exceptions.StreamError` with the reason + ``STREAM_ERROR_REASON_STORAGE_UNAVAILABLE``. Core then tries a call of Workflow + code again for up to 30 s. Every other reason reaches Workflow code on the first + try. Any other exception, or a ``StreamError`` with no reason, is a fault that a + retry can't fix. Workflow code gets it on the first try as a non-retryable + ``STREAM_ERROR_REASON_STORAGE_CORRUPT``, whose message tells the type of the + exception, and the Worker logs it. .. warning:: This API is experimental and subject to change. @@ -88,7 +197,8 @@ def limits(self) -> StreamBatchLimits: """The limits of one batch on this Store. The default is the largest batch that any Store may take. A Store that allows less - overrides this property, and refuses a larger batch itself. + overrides this property, and refuses a larger batch itself. Core refuses a larger + batch of Workflow code before the Store sees it. """ return StreamBatchLimits() @@ -137,6 +247,50 @@ async def describe_stream( """Give the status and the newest offset of a Stream.""" ... + def worker_activities(self) -> Sequence[Callable[..., Any]]: + """The Activities that a Worker registers for this Store when it runs Workflows. + + The default is one Activity, named for this Store, that + :py:meth:`call_from_workflow` runs as a Local Activity. A Store that overrides + one of the two methods overrides the other to match. + """ + return [_store_call_activity(self)] + + async def call_from_workflow( + self, + operation: str, + request: google.protobuf.message.Message, + options: StreamStoreCallOptions, + ) -> google.protobuf.message.Message: + """Answer a create or a describe of a Stream for Workflow code. + + It runs in Workflow code, so it does no I/O itself. The default runs the + Activity of :py:meth:`worker_activities` as a Local Activity for each operation. + It takes the timeouts and the retry policy of ``options``. A replay takes the + result from History and calls no method of this Store. + + Core makes a read, an append and a close of Workflow code itself, so this method + gets none of them. + + Args: + operation: The Nexus operation name, ``CreateStream`` or ``DescribeStream``. + request: The request message of ``temporalio.api.streamservice.v1``. + options: The :py:attr:`StreamConfig.workflow_call_options` of the config. + + Returns: + The response message of the operation. + """ + stream_operation = _STREAM_OPERATIONS[operation] + response = await temporalio.workflow.execute_local_activity( + _store_call_activity_name(self.name), + args=[operation, request.SerializeToString()], + result_type=bytes, + schedule_to_close_timeout=options.schedule_to_close_timeout, + start_to_close_timeout=options.start_to_close_timeout, + retry_policy=options.retry_policy, + ) + return stream_operation.response_type.FromString(response) + @dataclasses.dataclass(frozen=True) class _StreamOperation: @@ -200,6 +354,27 @@ def check_response( } """The operations of the Stream service, by their Nexus operation name.""" +_STORE_CALL_ACTIVITY = "__temporal_stream_store_call" + + +def _store_call_activity_name(store_name: str) -> str: + # The name carries the Store's name, so two Stores on one Worker don't clash. + return f"{_STORE_CALL_ACTIVITY}:{store_name}" + + +def _store_call_activity( + store: StreamStore, +) -> Callable[[str, bytes], Awaitable[bytes]]: + @temporalio.activity.defn(name=_store_call_activity_name(store.name)) + async def call(operation: str, request: bytes) -> bytes: + message = _STREAM_OPERATIONS[operation].request_type.FromString(request) + response = await _call_stream_store( + store, temporalio.activity.client().data_converter, operation, message + ) + return response.SerializeToString() + + return call + @dataclasses.dataclass(frozen=True) class StreamSelectContext: @@ -212,8 +387,12 @@ class StreamSelectContext: id: str """The id of the Stream.""" - target: StorageDriverActivityInfo | None = None - """The Activity that makes the call, or ``None`` for a plain client.""" + target: StorageDriverActivityInfo | StorageDriverWorkflowInfo | None = None + """The Workflow or Activity that makes the call, or ``None`` for a plain client. + + A call in an Activity has that Activity here. A Workflow's call runs the selector + in Workflow code, and has that Workflow here. + """ @dataclasses.dataclass(frozen=True) @@ -226,16 +405,68 @@ class StreamConfig: operation. It holds Stores and a selector, as :py:class:`temporalio.converter.ExternalStorage` holds drivers and a selector. + On a Worker whose client has no config, a Stream call of Workflow code raises + :py:class:`temporalio.exceptions.ApplicationError` with the ``type`` + ``StreamConfigError``, and an uncaught one fails the Workflow. Workflow code can + catch this error. A Workflow that catches it must not go on to make the same call + on a Worker that has a config, since the History then has no record of the first + call. + A ref carries the name of its Store, so a call on a known Stream goes to the Store of that name. A create can name its Store with ``store=``. A call whose ref names no Store goes to the Store that :py:attr:`store_selector` returns, or to the one Store when there is no selector. With more than one Store and no selector, such a call raises :py:class:`temporalio.exceptions.StreamConfigError`. - An Activity reaches the Stores of its Worker's client through - :py:func:`temporalio.activity.client`. Workflow code can't call a Stream in this - version. A Stream call of Workflow code raises ``RuntimeError``, which fails the - Workflow Task. + An append or a close of Workflow code writes as the Workflow, with the Workflow Task + that runs it as its epoch. So a Store that fences refuses a late write of a failed + Workflow Task. A ``producer=`` of another kind raises + :py:class:`temporalio.exceptions.StreamError` with the reason + ``STREAM_ERROR_REASON_INVALID``. Core keeps the appends and the close of one Workflow on one Stream in the order + of the calls. + + Core appends and closes for Workflow code itself, with no Activity. Core closes the + Stream only after the server took the Workflow Task of the close, so a failed task + closes nothing. A create and a describe of Workflow code run in the Store's Local + Activity. So Workflow code needs no Worker that polls for Activity Tasks. + + Workflow code reads with the same ``read``, ``read_pages`` and ``read_page`` as + a client. Core reads the Store inside the Workflow Task. A page that Workflow code gets in one activation has at most 256 Records, and + the rest come in the next activation with no wait. A loop that asks for its next page in the same + activation keeps one read open in Core. A read that no code waits on closes before + the activation completes, so it never holds a Workflow Task open. + + .. code-block:: python + + async for record in events.read(wait=timedelta(seconds=10)): + if record.body.kind == "tool_call": + tools.append( + workflow.start_activity( + run_tool, + record.body.tool_call, + start_to_close_timeout=timedelta(minutes=1), + ) + ) + + The loop ends after the page that is ``done``, when the Stream is closed and the + read gave its last Record. With a ``wait``, a quiet Stream gives empty pages, and + the loop goes on. So a loop on a Stream that can be quiet for long uses + ``read_pages`` and leaves after an empty page. A read has these costs. + + * A call keeps its Workflow Task open while it waits for its page. A Signal, an + Update, a cancel or an Activity result reaches the Workflow at the next Workflow + Task, at the heartbeat of the task at the latest. That is about 8 s with the + default task timeout of 10 s. + * The calls of one Workflow Task write one marker in History, whatever the number of + Records. + * With the default :py:attr:`StreamReplaySource.STORE`, History holds no Record. A + replay reads the Records from the Store again, so they must stay in the Store for + as long as the Workflow can replay. + * A replay on a Worker with no cache reads the Store again on every Workflow Task. + + Core calls a Store written in Python on the Worker's event loop, with the same rules. + :py:class:`StreamStore` says what such a Store must do for Workflow code. Code in a + Query handler or an Update validator can't read. .. warning:: This API is experimental and subject to change. @@ -246,7 +477,8 @@ class StreamConfig: A ref carries the :py:attr:`StreamStore.name` of its Store, and a call looks the Store up by that name. With more than one Store and no - :py:attr:`store_selector`, each call must name its Store. + :py:attr:`store_selector`, each call must name its Store. A Worker serves every + Store of its client. """ store_selector: Callable[[StreamSelectContext], StreamStore] | None = None @@ -257,8 +489,62 @@ class StreamConfig: such a call goes to that Store. When it is ``None`` and there is more than one Store, such a call raises :py:class:`temporalio.exceptions.StreamConfigError`. - A Stream that code builds from an id only goes through the selector on each - call, so the selector must give the same Store for the same id. + For a call of Workflow code, the selector runs in Workflow code, also on a + replay. So it must give the same Store for the same context. The call of the chosen + Store is in History, and another answer on a replay is a non-determinism error. A Stream that code builds from an id only goes through the + selector on each call, so the selector must also give the same Store for the same + id. + """ + + workflow_call_options: StreamStoreCallOptions = dataclasses.field( + default_factory=StreamStoreCallOptions + ) + """Options of a create and a describe of Workflow code, for every Store of this config. + + :py:meth:`StreamStore.call_from_workflow` gets them. The default runs the Store's + Local Activity with their timeouts and retry policy. + """ + + workflow_read_replay_from: StreamReplaySource = dataclasses.field( + default=StreamReplaySource.STORE + ) + """Where a replay of a read of Workflow code gets the Records of the read. + + It applies to every read of Workflow code on a Worker whose client has this config, + and decides what a new read records in History. :py:attr:`StreamReplaySource.STORE`, + the default, puts only the range of offsets, the count and a digest of the Records + into History, so a replay reads them from the Store again. The Records must then stay + in the Store for as long as the Workflow can replay. The option + :py:attr:`StreamReplaySource.HISTORY` puts the Records with their bodies into + History, so a replay needs no Store. A replay follows the marker in History and not + this setting, so a Workflow that is running replays after a change of it. + """ + + workflow_read_ahead_bytes: int | None = None + """The most bytes of one read's Records that wait in the Worker for Workflow code. + + A read of Workflow code stops asking the Store while this many bytes of its Records + wait in the Worker's memory for Workflow code to take them. The read goes + on when Workflow code takes Records. The first page of a read always comes. ``None`` + takes Core's default of 4 MiB. Set a larger value for large Records, and a smaller one + for a Worker with little memory, since each open read can hold this much. + """ + + workflow_read_page_records: int | None = None + """The most Records of one read that Workflow code gets in one activation. + + Core keeps the Records over it and gives them in the next activation, with no wait. + ``None`` takes Core's default of 256. The value must be 1 to 1000. A replay gives + each page as History has it, whatever this value is. + """ + + workflow_read_min_gap: timedelta | None = None + """The least time between two batches of one read of Workflow code. + + A read gives its first batch immediately. The Records that come in the gap after a + batch come together in the next batch. This trades latency for fewer activations of + Workflow code, since a Record can wait up to one gap. ``None`` and zero mean no gap, + the default. The value must be zero to 1 s. A replay ignores it. """ _store_map: dict[str, StreamStore] = dataclasses.field( @@ -270,8 +556,30 @@ def __post_init__(self) -> None: """Check the Stores and build the name-keyed map of them. Raises :py:class:`temporalio.exceptions.StreamConfigError` if there is no - Store, or if two Stores share a name. + Store, or if two Stores share a name. Raises ``ValueError`` if + :py:attr:`workflow_read_ahead_bytes` is below 1, if + :py:attr:`workflow_read_page_records` is not 1 to 1000, or if + :py:attr:`workflow_read_min_gap` is not zero to 1 s. """ + if ( + self.workflow_read_ahead_bytes is not None + and self.workflow_read_ahead_bytes < 1 + ): + raise ValueError("workflow_read_ahead_bytes must be at least 1") + if self.workflow_read_page_records is not None and not ( + 1 <= self.workflow_read_page_records <= 1000 + ): + raise ValueError( + "workflow_read_page_records must be 1 to 1000, " + f"got {self.workflow_read_page_records}" + ) + if self.workflow_read_min_gap is not None and not ( + timedelta(0) <= self.workflow_read_min_gap <= timedelta(seconds=1) + ): + raise ValueError( + "workflow_read_min_gap must be zero to 1 second, " + f"got {self.workflow_read_min_gap}" + ) if not self.stores: raise StreamConfigError("StreamConfig.stores must have at least one Store") store_map: dict[str, StreamStore] = {} @@ -316,7 +624,7 @@ def _get_store_by_name(self, name: str) -> StreamStore: def _store_for( self, ref: temporalio.nexus.system.stream_service.StreamRef[Any], - target: StorageDriverActivityInfo | None, + target: StorageDriverActivityInfo | StorageDriverWorkflowInfo | None, ) -> StreamStore: if ref.store: return self._get_store_by_name(ref.store) @@ -372,15 +680,6 @@ def _stream_service() -> nexusrpc.ServiceDefinition: _STREAM_SERVICE = _stream_service() -def _workflow_stream_call_error() -> RuntimeError: # pyright: ignore[reportUnusedFunction] - # Not an `ApplicationError`, so it fails the Workflow Task and not the Workflow. A - # Worker with code that has no Stream call can then finish the Workflow. - return RuntimeError( - "Workflow code can't call a Stream in this version. Call it from an Activity " - "through activity.client()." - ) - - async def _call_stream_store( # pyright: ignore[reportUnusedFunction] store: StreamStore, data_converter: temporalio.converter.DataConverter, diff --git a/temporalio/contrib/streams/README.md b/temporalio/contrib/streams/README.md index 4c5a8e46a4..1478e8f78c 100644 --- a/temporalio/contrib/streams/README.md +++ b/temporalio/contrib/streams/README.md @@ -1,14 +1,14 @@ # Streams helpers -> This package is experimental, and its API can change. +> This package is experimental. Its API can change in a later release. -`temporalio.contrib.streams` holds helpers that sit on top of the Streams API of the SDK. They -run in a client or an Activity, and take a client `Stream`. +`temporalio.contrib.streams` holds helpers that sit on top of the Streams API of the SDK. ## `BufferedAppender` `BufferedAppender` batches the appends of one Stream. `append` returns immediately, and the -appender writes the buffered Records as one batch. A batch waits at most `max_delay`. +appender writes the buffered Records as one batch. A batch waits at most `max_delay`. The +appender is for client and Activity code. Workflow code passes its Records to one `append` call. ```python from temporalio.contrib.streams import BufferedAppender @@ -40,3 +40,151 @@ async def write_parts(ref: StreamRef[str], parts: list[str]) -> None: The retry's Records continue the first attempt's epoch, so a reader does not take them as a replacement of the first attempt's Records. + +## Reads and appends in Workflow code + +This section says how Workflow code reads, appends and closes a Stream. On every Store, such +as the Redis Store, Core reads, appends and closes a Stream itself, with no Activity. Workflow +code reads with the same `read`, `read_pages` and `read_page` as a client. It gets the same +pages, but each page has at most 256 Records. + +Here the Workflow reads the events of a model's answer while the model Activity still runs, +and starts each tool as its call arrives. + +```python +events = workflow.get_stream(events_ref, body_type=ModelEvent) +status = workflow.get_stream(status_ref, body_type=str) +tools = [] +async for record in events.read(wait=timedelta(seconds=10)): + event = record.body + if event.kind == "tool_call": + # The tool starts now, before the model ends its answer. + tools.append( + workflow.start_activity( + run_tool, event.tool_call, start_to_close_timeout=timedelta(minutes=1) + ) + ) + await status.append(f"Running {event.tool_call.name}") +results = await asyncio.gather(*tools) +``` + +The loop ends after the page that is `done`, when the Stream is closed and the read gave its +last Record. With no `wait`, a call gives what is in the Stream, and the loop ends at the first +empty page. With a `wait`, a call on a quiet Stream gives an empty page after the wait, and +the loop asks again. So a loop on a Stream that can be quiet for long uses `read_pages` and +leaves after an empty page. A later loop from the `last_offset` of that page goes on there. + +```python +last = None +async for page in tokens.read_pages(from_offset=last, wait=timedelta(seconds=30)): + for record in page.records: + await shown.append(record.body.text) + last = page.last_offset + if not page.records: + # The Stream was quiet for 30 s, or it is done. + break +``` + +Core keeps one read open while a loop asks for its next page in the same activation. A loop +that awaits something else between two pages, as the `append` above does, lets the read close +before the activation completes, and its next page starts a new read. A read that no code +waits on never holds a Workflow Task open. + +The calls of one Workflow Task write one marker in History, whatever the number of Records. A +read has three other costs. + +- A call keeps its Workflow Task open while it waits for its page. A Signal, an Update, a + cancel or an Activity result reaches the Workflow at the next Workflow Task, at the + heartbeat of the task at the latest. That is about 8 s with the default task timeout of + 10 s. A call with a `wait` on a quiet Stream holds its task open for the whole wait. +- By default, `StreamReplaySource.STORE`, the marker holds a range of offsets, a count and a + digest, and no Record. A replay reads the Store again, so the Records must stay in the + Store for as long as the Workflow can replay. A replay whose Store stays unavailable for + 5 s fails only its Workflow Task, with `StreamReplayUnavailable`, and never the Workflow. + The option + `workflow_read_replay_from=StreamReplaySource.HISTORY` of `StreamConfig` puts the Records + into the marker, so a replay needs no Store. Then one Workflow Task's marker holds at most + 1 MiB of Records. Past that, the task completes and the read goes on in the next one. +- A replay on a Worker with no cache reads the Store again on every Workflow Task. + +While Workflow code is busy, a read holds at most 4 MiB of its Records in the Worker, then +waits until the code takes them. `workflow_read_ahead_bytes` of `StreamConfig` changes that +bound, for large Records or a Worker with little memory. A read gives Workflow code its Records +as soon as no activation of the Workflow is pending. A page that Workflow code gets in one +activation has at most 256 Records, and the rest come in the next activation with no wait. +`workflow_read_page_records` changes that cap, from 1 to 1000. + +`workflow_read_min_gap` of `StreamConfig` sets the least time between two batches of one read. +It has no gap by default, and it must be at most 1 s. A read gives its first batch immediately. +The Records that come in the gap come together in the next batch. This trades latency for fewer +activations of Workflow code. A replay ignores both settings. + +An append and a close of Workflow code always write as the Workflow, with the Workflow Task as +their epoch. A `producer=` of another kind raises `StreamError` with the reason +`STREAM_ERROR_REASON_INVALID`. An append runs inside its Workflow Task and records a marker with no body. An +append is at-least-once: when a Worker crashes or its run is evicted before the task with the +append completes, the rerun of that task writes a second copy with the epoch of the new task. + +Core tries a read, an append or a close again while its Store answers `StorageUnavailable`. +It waits from 100 ms up to 2 s between tries, for at most 30 s. Only then does Workflow code +get the retryable `StreamError`. Every other reason raises on the first try. An append of +Workflow code always carries a `sequence`, so the Store writes its Records once, also when a +try reached the Store but its answer got lost. + +A close spans two Workflow Tasks. The task that calls `close` records only the intent. Core +closes the Stream after the server took that task, and the next task gives the result. So a +failed task closes nothing, and its retry can still append. A Workflow that awaits its close +never completes in the task that asked for it. Await the close, also in a `finally` block. + +```python +stream = await workflow.create_stream(id=stream_id, body_type=str) +try: + await stream.append("a") +finally: + await stream.close() +``` + +A close that the code doesn't await still happens when the Workflow ends in the same task, +after the server took that task. A graceful shutdown of the Worker waits for such a close, for +at most 10 s. Only a Worker that dies there, or a Store that doesn't answer in that time, loses +the close, and the Stream stays open. + +A Store is a subclass of `StreamStore`, written in Python. Core makes a read, an append and a +close of Workflow code on it. It calls the Store's method as a task on the Worker's event loop, +never on a Workflow thread. For Workflow code, a Store must do these things. + +- Each method is `async` and never blocks the event loop. +- `read_records` waits up to the request's `wait` when it has no Record after `from_offset`, and + an append or a close wakes it. The wait must end on a cancel, since Core cancels the task of a + read that it stops. +- Each Record has an offset of 1 to 1024 bytes, and offsets compare in order as bytes. A page + that gives the last Record of a closed Stream is `done`. +- `append_records` writes a batch that comes again with the same Producer, epoch and `sequence` + once. It can refuse a lower epoch of a Producer with `STREAM_ERROR_REASON_FENCED`, and it + doesn't have to. +- `close_stream` of a closed Stream succeeds. +- The Store keeps the Records with their offsets for as long as a replay reads them. +- An outage raises `StreamError` with `STREAM_ERROR_REASON_STORAGE_UNAVAILABLE`, and Core tries + the call again for up to 30 s. Any other exception is not tried again. Workflow code gets it + as a non-retryable `STREAM_ERROR_REASON_STORAGE_CORRUPT`, with its type in the message. + +## Metrics + +Core records these metrics for the Direct calls of Workflow code. Their prefix is the metric +prefix of the `Runtime`, which is `temporal_` by default. Each metric has the attribute +`stream_store` with the Store of the call, next to the `namespace`, `task_queue` and +`workflow_type` that a Local Activity metric has. A replay that takes a result from a marker +counts nothing. + +| Metric | What it counts | +| --- | --- | +| `temporal_stream_read_total` | Reads that Core started | +| `temporal_stream_read_failed` | Reads that ended with a failure | +| `temporal_stream_read_records` | Records that reads gave to Workflow code | +| `temporal_stream_append_total`, `_failed` | Appends that Core sent, and those that failed | +| `temporal_stream_append_latency` | From the start of an append in Core to its result | +| `temporal_stream_close_total`, `_failed` | Closes that Core sent, and those that failed | +| `temporal_stream_close_latency` | From the start of a close in Core to its result | +| `temporal_stream_close_abandoned` | Closes that a graceful shutdown gave up after 10 s | +| `temporal_stream_replay_read_total`, `_failed` | Marker entries that a replay read again | +| `temporal_stream_replay_read_latency` | The time of one such read and its check | diff --git a/temporalio/contrib/streams/__init__.py b/temporalio/contrib/streams/__init__.py index 95318f94d1..b3680baa79 100644 --- a/temporalio/contrib/streams/__init__.py +++ b/temporalio/contrib/streams/__init__.py @@ -1,9 +1,8 @@ """Helpers that sit on top of the Streams API. -:py:class:`BufferedAppender` batches the appends of one client Stream. -:py:func:`current_producer` gives the Producer that an append of the calling code sends by -default, and :py:func:`as_producer` gives back a Producer that code stored. They run in a -client or an Activity. +:py:class:`BufferedAppender` batches the appends of one Stream. It is for client and Activity +code. :py:func:`current_producer` gives the Producer that an append of the calling code sends by +default, and :py:func:`as_producer` gives back a Producer that code stored. .. warning:: This package is experimental and may change in future versions. diff --git a/temporalio/contrib/streams/_appender.py b/temporalio/contrib/streams/_appender.py index 8625da3d80..1c49172556 100644 --- a/temporalio/contrib/streams/_appender.py +++ b/temporalio/contrib/streams/_appender.py @@ -14,6 +14,7 @@ from typing_extensions import Self import temporalio.nexus.system.stream_service.client +import temporalio.workflow BodyT = TypeVar("BodyT") @@ -32,8 +33,9 @@ class BufferedAppender(Generic[BodyT]): Records appended during a write wait for the next batch, and they keep their order. A batch holds at most ``max_batch_records`` Records. - It takes a client Stream, in a client or an Activity. It writes the buffered Records at - most ``max_delay`` after the first of them, or immediately when they fill a batch. + The appender writes the buffered Records at most ``max_delay`` after the first of them, or + immediately when they fill a batch. It is for client and Activity code. Workflow code can't + use it, and passes its Records to one ``append`` call of the Stream. Use it in ``async with``, which writes the remaining Records at the end of the block, or await :py:meth:`flush`. Without ``async with`` or a :py:meth:`flush`, the Records still in @@ -62,7 +64,7 @@ def __init__( """Create an appender. It starts no write until the first append. Args: - stream: The client Stream to append to. + stream: The Stream to append to, from a client or from the client of an Activity. max_delay: The longest time that a Record waits in the buffer before its batch starts to write. It must not be negative. ``None`` means 100 milliseconds. max_batch_records: The most Records of one batch, from 1 to 1,000. Set it to the @@ -71,9 +73,15 @@ def __init__( ``None`` means 1,000. Raises: + RuntimeError: The appender is built in Workflow code. ValueError: ``max_delay`` is negative, or ``max_batch_records`` is below 1 or above 1,000. """ + if temporalio.workflow.in_workflow(): + raise RuntimeError( + "BufferedAppender is for client and Activity code. In Workflow code, pass " + "the Records to one append call." + ) if max_delay is not None and max_delay < timedelta(): raise ValueError("max_delay must not be negative") if max_batch_records is None: @@ -89,6 +97,7 @@ def __init__( self._buffer: list[BodyT] = [] # When the oldest buffered Record arrived, so its batch starts at most max_delay later. self._first_at = 0.0 + # The timer that starts the next batch. self._timer: asyncio.Handle | None = None self._writing: asyncio.Task[None] | None = None self._appended = 0 diff --git a/temporalio/contrib/streams/_producer.py b/temporalio/contrib/streams/_producer.py index 51518d942c..f94b0d1baf 100644 --- a/temporalio/contrib/streams/_producer.py +++ b/temporalio/contrib/streams/_producer.py @@ -3,7 +3,6 @@ from __future__ import annotations import temporalio.api.stream.v1 -import temporalio.client._stream_store import temporalio.converter._payload_converter import temporalio.nexus.system._support import temporalio.workflow @@ -14,23 +13,23 @@ def current_producer() -> Producer: """Give the Producer that an append or a close of the calling code sends by default. An ``append`` or a ``close`` with no ``producer=`` sends this Producer. In an - Activity or a Local Activity it is the Producer of the current attempt. In any other - code it is the client Producer of this process. It does no I/O. + Activity or a Local Activity it is the Producer of the current attempt. In Workflow + code it is the Workflow's Producer. In any other code it is the client Producer of + this process. It does no I/O. + + On a Store, an append or a close of Workflow code writes as this Producer, with the + Workflow Task that runs it as its epoch. An Activity can keep the result and pass it to a later ``append`` or ``close`` with ``producer=``. That call then writes as this Producer with its epoch, also in a later attempt. Heartbeat details give it back as its proto message, so pass it through :py:func:`as_producer` first. - Raises: - RuntimeError: It runs in Workflow code, which can't call a Stream in this - version. - .. warning:: This API is experimental and subject to change. """ if temporalio.workflow.in_workflow(): - raise temporalio.client._stream_store._workflow_stream_call_error() + return temporalio.nexus.system._support.workflow_producer() return temporalio.nexus.system._support.default_producer() diff --git a/temporalio/exceptions.py b/temporalio/exceptions.py index 3f2df0fc9a..6954c08cde 100644 --- a/temporalio/exceptions.py +++ b/temporalio/exceptions.py @@ -212,13 +212,15 @@ def category(self) -> ApplicationErrorCategory: class StreamError(ApplicationError): """A failed Stream call. ``reason`` tells the cases apart. - One class covers every failed call in client and Activity code, so match on - ``reason`` and not on the type. The reason is a ``StreamErrorReason``, such as + One class covers every failed call in client, Activity and Workflow code, so match + on ``reason`` and not on the type. The reason is a ``StreamErrorReason``, such as ``STREAM_ERROR_REASON_NOT_FOUND`` (nobody created the Stream), ``STREAM_ERROR_REASON_CLOSED`` (the Stream is closed), or ``STREAM_ERROR_REASON_ALREADY_EXISTS`` (a strict create of a Stream that exists). A Store outage, ``STREAM_ERROR_REASON_STORAGE_UNAVAILABLE``, is the one retryable - reason. + reason. Core tries a read, an append or a close of Workflow code again while the + Store is unavailable. It waits from 100 ms up to 2 s between tries, and Workflow + code gets this reason only after 30 s. Every other reason raises on the first try. .. warning:: This API is experimental and subject to change. @@ -249,6 +251,32 @@ def reason(self) -> temporalio.api.enums.v1.StreamErrorReason.ValueType: return self._reason +def _stream_error_from_reason(reason: int, message: str) -> StreamError: + if reason not in temporalio.api.enums.v1.StreamErrorReason.values(): + # A reason that this SDK doesn't know gives the unspecified reason. + reason = _STREAM_ERROR_REASON_UNSPECIFIED + return StreamError( + message, reason=temporalio.api.enums.v1.StreamErrorReason.ValueType(reason) + ) + + +def _stream_error_from_failure( # pyright: ignore[reportUnusedFunction] + err: BaseException, +) -> StreamError | None: + # A Local Activity or a Nexus operation hands back a plain application error, so the + # error is rebuilt from its type and details. + if isinstance(err, StreamError): + return err + if not isinstance(err, ApplicationError) or err.type != "StreamError": + return None + reason: int = _STREAM_ERROR_REASON_UNSPECIFIED + for detail in err.details: + if isinstance(detail, temporalio.api.stream.v1.StreamErrorDetails): + reason = detail.reason + break + return _stream_error_from_reason(reason, err.message) + + class StreamConfigError(RuntimeError): """A Stream call can't find its Store, because of how the Stores are set up. @@ -257,12 +285,48 @@ class StreamConfigError(RuntimeError): * A Stream call is made on a client with no Stream config. This release has no Streams on the server, so every Stream call needs a Store. + * A Workflow's Stream call has a ref that names no Store, on a Worker whose + client has more than one Store and no selector. This fails the Workflow Task. * A :py:class:`temporalio.client.StreamConfig` has no Store, or has two Stores with one name. * A call names no Store on a config with more than one Store and no selector. * A ref names a Store that the config of the client doesn't have. * The selector returns a Store that isn't one of the config's Stores. * A :py:class:`temporalio.client.StreamStore` is given in ``plugins``. + * A Workflow's Stream call has a ref that names a Store that the config of the + Worker's client doesn't have. This fails the Workflow Task. + + A Workflow's Stream call on a Worker whose client has no Stream config doesn't + raise this class. Workflow code gets an :py:class:`ApplicationError` with the + ``type`` ``StreamConfigError`` and ``non_retryable`` set, so an uncaught one fails + the Workflow. Workflow code can catch this error. A Workflow that catches it must not go + on to make the same call on a Worker that has a config, since the History then has + no record of the first call. + + .. warning:: + This API is experimental and subject to change. + """ + + +class StreamReplayError(TemporalError): + """A replay could not read the Records of a read of Workflow code from its Store again. + + A read under the default :py:attr:`temporalio.client.StreamReplaySource.STORE` keeps + no Record in History, so a replay reads the Records from the Store again. When the + Store doesn't have them, for example after the Stream's Retention, or they differ from + what History says, the replay can't go on. + + By default that fails only the Workflow Task, so a later attempt works once the + Records are back. With this class in ``workflow_failure_exception_types`` of the + Worker, or in ``failure_exception_types`` of ``@workflow.defn``, the Workflow fails + instead. Its failure is an :py:class:`ApplicationError` with the ``type`` + ``StreamReplayError``, and Core raises no instance of this class. + + A Store that doesn't answer is not this error, since the Records may still be + there. A replay tries the Store again for 5 s. If it stays unavailable, only the + Workflow Task fails, with a message that starts with ``StreamReplayUnavailable``. + That holds also with this class in ``workflow_failure_exception_types`` or + ``failure_exception_types``, so a later attempt of the task can read the Records. .. warning:: This API is experimental and subject to change. diff --git a/temporalio/nexus/system/stream_service/workflow/_resources/stream.py b/temporalio/nexus/system/stream_service/workflow/_resources/stream.py index df548d2528..3786698d1a 100644 --- a/temporalio/nexus/system/stream_service/workflow/_resources/stream.py +++ b/temporalio/nexus/system/stream_service/workflow/_resources/stream.py @@ -2,6 +2,8 @@ from __future__ import annotations +import collections.abc +import datetime import types import typing @@ -11,6 +13,7 @@ import temporalio.nexus.system._support as _support import temporalio.workflow +from ..._poll import PollIterator from ...models import ( AppendRecord, AppendRecordsRequest, @@ -20,6 +23,9 @@ DescribeStreamRequest, DescribeStreamResponse, Producer, + ReadRecordsRequest, + ReadRecordsResponse, + StreamRecord, StreamRef, ) @@ -86,6 +92,103 @@ async def close( handle = await _close_stream(request) await handle + async def read_page( + self, + from_offset: bytes | None = None, + max_records: int | None = None, + wait: datetime.timedelta | None = None, + max_bytes: int | None = None, + ) -> ReadRecordsResponse[BodyT]: + """Read one page of Records. The poll iterator reads them all.""" + request = ReadRecordsRequest( + ref=self.ref, + from_offset=from_offset, + max_records=max_records, + wait=wait, + max_bytes=max_bytes, + ) + handle = await _read_records(request, self.body_type) + return await handle + + async def read_pages( + self, + from_offset: bytes | None = None, + max_records: int | None = None, + wait: datetime.timedelta | None = None, + max_bytes: int | None = None, + ) -> collections.abc.AsyncGenerator[ReadRecordsResponse[BodyT], None]: + """Yield each response of ``read_page`` until the poll helper stops. + + Read one page of Records. The poll iterator reads them all. + + Args: + from_offset: Read after this offset. Empty is the beginning. A too high + offset fails. + max_records: About this many Records per page, 1 to 1000. + wait: How long to wait for a new Record before an empty answer. At most 60 + s. + max_bytes: The maximum Record bytes in a page. A larger Record is still + returned alone. + """ + request = ReadRecordsRequest( + ref=self.ref, + from_offset=from_offset, + max_records=max_records, + wait=wait, + max_bytes=max_bytes, + ) + while True: + handle = await _read_records(request, self.body_type) + response = await handle + next_request = _support.next_read_request(request, response) + yield response + if next_request is None: + return + request = next_request + + def read( + self, + from_offset: bytes | None = None, + max_records: int | None = None, + wait: datetime.timedelta | None = None, + max_bytes: int | None = None, + ) -> PollIterator[ReadRecordsRequest[BodyT], StreamRecord[BodyT]]: + """Yield each item of ``read_pages``, page by page. + + ``next_request`` resumes after the last item that the iterator yielded. It is + ``None`` after the last page. + + Read one page of Records. The poll iterator reads them all. + + Args: + from_offset: Read after this offset. Empty is the beginning. A too high + offset fails. + max_records: About this many Records per page, 1 to 1000. + wait: How long to wait for a new Record before an empty answer. At most 60 + s. + max_bytes: The maximum Record bytes in a page. A larger Record is still + returned alone. + """ + request = ReadRecordsRequest( + ref=self.ref, + from_offset=from_offset, + max_records=max_records, + wait=wait, + max_bytes=max_bytes, + ) + return PollIterator( + request, + self.read_pages( + from_offset=from_offset, + max_records=max_records, + wait=wait, + max_bytes=max_bytes, + ), + lambda response: response.records, + _support.next_read_request, + _support.resume_read_request, + ) + async def describe( self, ) -> DescribeStreamResponse: @@ -164,6 +267,21 @@ async def _close_stream( ) +async def _read_records( + request: ReadRecordsRequest[BodyT], + body_type: type[BodyT], +) -> temporalio.workflow.NexusOperationHandle[ReadRecordsResponse[BodyT],]: + nexus_client = temporalio.workflow.create_nexus_client( + service="temporal.api.streamservice.v1.StreamService", + endpoint=temporalio.nexus.system.TEMPORAL_SYSTEM_ENDPOINT, + ) + return await nexus_client.start_operation( + operation="ReadRecords", + input=request, + output_type=ReadRecordsResponse[body_type], + ) + + async def _describe_stream( request: DescribeStreamRequest[BodyT], ) -> temporalio.workflow.NexusOperationHandle[DescribeStreamResponse,]: diff --git a/temporalio/worker/_command_aware_visitor.py b/temporalio/worker/_command_aware_visitor.py index 888c06f298..8a42bc9a8a 100644 --- a/temporalio/worker/_command_aware_visitor.py +++ b/temporalio/worker/_command_aware_visitor.py @@ -16,8 +16,13 @@ ResolveNexusOperationStart, ResolveRequestCancelExternalWorkflow, ResolveSignalExternalWorkflow, + ResolveStreamAppend, + ResolveStreamClose, + ResolveStreamRead, + StreamReadEnd, ) from temporalio.bridge.proto.workflow_commands.workflow_commands_pb2 import ( + AppendToStream, CompleteWorkflowExecution, FailWorkflowExecution, ScheduleActivity, @@ -37,6 +42,8 @@ class CommandInfo: command_type: CommandType.ValueType command_seq: int update_protocol_instance_id: str | None = None + stream: bool = False + """Whether the payloads are the bodies of a Stream append or the Records of a read.""" current_command_info: contextvars.ContextVar[CommandInfo | None] = ( @@ -57,6 +64,7 @@ def __init__( skip_search_attributes: bool = False, skip_headers: bool = False, concurrency_limit: int = 1, + skip_stream_bodies: bool = False, ) -> None: """Creates a new command-aware payload visitor. @@ -65,12 +73,53 @@ def __init__( skip_headers: If True, headers are not visited. concurrency_limit: Maximum number of payload visits that may run concurrently during a single call to visit(). Defaults to 1. + skip_stream_bodies: If True, the bodies of a Stream append are not + visited. """ super().__init__( skip_search_attributes=skip_search_attributes, skip_headers=skip_headers, concurrency_limit=concurrency_limit, ) + self.skip_stream_bodies = skip_stream_bodies + + # A Store keeps the bodies of a Workflow's append as a client and an Activity write + # them: with the codec, with no serialization context and with no external storage. So + # every reader decodes them. Each body is one Record, so the codec gets each body alone, + # as for those paths. + async def _visit_coresdk_workflow_commands_AppendToStream( + self, fs: VisitorFunctions, o: AppendToStream + ) -> None: + if self.skip_stream_bodies: + return + with current_command(CommandType.COMMAND_TYPE_UNSPECIFIED, o.seq, stream=True): + for body in o.bodies: + await self._visit_temporal_api_common_v1_Payload(fs, body) + + # The Records of a read come from any writer of the Stream, so the codec decodes them + # with no serialization context, as a client and an Activity do. + async def _visit_coresdk_workflow_activation_ResolveStreamRead( + self, fs: VisitorFunctions, o: ResolveStreamRead + ) -> None: + with current_command(CommandType.COMMAND_TYPE_UNSPECIFIED, o.seq, stream=True): + await super()._visit_coresdk_workflow_activation_ResolveStreamRead(fs, o) + + # Core builds the failure of a Stream read, append or close itself, with no codec, so no + # pass may decode it. + async def _visit_coresdk_workflow_activation_StreamReadEnd( + self, fs: VisitorFunctions, o: StreamReadEnd + ) -> None: + pass + + async def _visit_coresdk_workflow_activation_ResolveStreamAppend( + self, fs: VisitorFunctions, o: ResolveStreamAppend + ) -> None: + pass + + async def _visit_coresdk_workflow_activation_ResolveStreamClose( + self, fs: VisitorFunctions, o: ResolveStreamClose + ) -> None: + pass # Workflow commands with payloads async def _visit_coresdk_workflow_commands_CompleteWorkflowExecution( @@ -218,6 +267,7 @@ def current_command( command_seq: int, *, update_protocol_instance_id: str | None = None, + stream: bool = False, ) -> Iterator[None]: """Context manager for setting command info.""" token = current_command_info.set( @@ -225,6 +275,7 @@ def current_command( command_type=command_type, command_seq=command_seq, update_protocol_instance_id=update_protocol_instance_id, + stream=stream, ) ) try: diff --git a/temporalio/worker/_replayer.py b/temporalio/worker/_replayer.py index 62a27b29cf..17e630fc4a 100644 --- a/temporalio/worker/_replayer.py +++ b/temporalio/worker/_replayer.py @@ -22,6 +22,13 @@ from ..common import HeaderCodecBehavior from ._interceptor import Interceptor +from ._stream_store import ( + _bridge_read_ahead_bytes, + _bridge_read_min_gap, + _bridge_read_page_records, + _core_stream_runtimes, + _WorkflowStreamStore, +) from ._worker import load_default_build_id from ._workflow import _WorkflowWorker from ._workflow_instance import ( @@ -56,6 +63,7 @@ def __init__( runtime: temporalio.runtime.Runtime | None = None, disable_safe_workflow_eviction: bool = False, header_codec_behavior: HeaderCodecBehavior = HeaderCodecBehavior.NO_CODEC, + stream_config: temporalio.client.StreamConfig | None = None, ) -> None: """Create a replayer to replay workflows from history. @@ -64,6 +72,22 @@ def __init__( the replayer that were passed to the worker when the workflow originally ran. + ``stream_config`` takes the Stream config of the client that the + worker had, so a replay makes the same commands. A history that records a + create or a describe, which run in a Store's Local Activity, needs it. With no + Stream config, Workflow code gets an error in place of that + call, and the replay fails with a nondeterminism error. A replay calls no method + of its Stores from Python. An append or a close needs no Stream + config, and a replay closes nothing. A read of Workflow code that ran under the + option ``HISTORY`` of ``workflow_read_replay_from`` needs none either. That holds + also when the ref has no Store, since Core takes the Store from the history. A + read that ran under the default, ``StreamReplaySource.STORE``, has no Records in + the history. Core reads them from the Store again, so the replay needs this + config and Stores that still have the Records. Without them the replay fails + with a :py:class:`RuntimeError` whose message has + ``StreamReplayError``, the Stream and the cause. A Store that stays unavailable + for 5 s gives ``StreamReplayUnavailable`` in place of ``StreamReplayError``. + Note, unlike the worker, for the replayer the workflow_task_executor will default to a new thread pool executor with no max_workers set that will be shared across all replay calls and never explicitly shut down. @@ -86,6 +110,7 @@ def __init__( runtime=runtime, disable_safe_workflow_eviction=disable_safe_workflow_eviction, header_codec_behavior=header_codec_behavior, + stream_config=stream_config, ) self._initial_config = self._config.copy() self._default_workflow_logic_flags = set(_DEFAULT_ENABLED_WORKFLOW_LOGIC_FLAGS) @@ -287,6 +312,9 @@ def on_eviction_hook( default_workflow_logic_flags=frozenset( self._default_workflow_logic_flags ), + stream_store=_WorkflowStreamStore.from_config( + self._config.get("stream_config") + ), ) external_storage = data_converter.external_storage storage_driver_types = ( @@ -295,6 +323,12 @@ def on_eviction_hook( else set() ) + # A replay reads a Store written in Python on this loop. + core_stream_runtimes = _core_stream_runtimes( + self._config.get("stream_config"), runtime + ) + core_stream_runtimes.bind(asyncio.get_running_loop()) + # Create bridge worker bridge_worker, pusher = temporalio.bridge.worker.Worker.for_replay( runtime._core_runtime, @@ -307,6 +341,10 @@ def on_eviction_hook( # per workflow type nondeterminism_as_workflow_fail=workflow_worker.nondeterminism_as_workflow_fail(), nondeterminism_as_workflow_fail_for_types=workflow_worker.nondeterminism_as_workflow_fail_for_types(), + stream_replay_as_workflow_fail=workflow_worker.stream_replay_as_workflow_fail(), + stream_replay_as_workflow_fail_for_types=( + workflow_worker.stream_replay_as_workflow_fail_for_types() + ), # All values below are ignored but required by Core max_cached_workflows=2, tuner=temporalio.bridge.worker.TunerHolder( @@ -356,6 +394,16 @@ def on_eviction_hook( (plugin.name(), plugin.version()) for plugin in self.plugins ], storage_drivers=storage_driver_types, + stream_runtimes=core_stream_runtimes.runtimes, + stream_read_ahead_bytes=_bridge_read_ahead_bytes( + self._config.get("stream_config") + ), + stream_read_page_records=_bridge_read_page_records( + self._config.get("stream_config") + ), + stream_read_min_gap=_bridge_read_min_gap( + self._config.get("stream_config") + ), ), ) bridge_worker_scope = bridge_worker @@ -435,6 +483,7 @@ class ReplayerConfig(TypedDict, total=False): runtime: temporalio.runtime.Runtime | None disable_safe_workflow_eviction: bool header_codec_behavior: HeaderCodecBehavior + stream_config: temporalio.client.StreamConfig | None @dataclass(frozen=True) diff --git a/temporalio/worker/_stream_store.py b/temporalio/worker/_stream_store.py new file mode 100644 index 0000000000..9c14707b65 --- /dev/null +++ b/temporalio/worker/_stream_store.py @@ -0,0 +1,627 @@ +"""How Workflow code reaches the Stream Stores of its Worker's client.""" + +from __future__ import annotations + +import asyncio +import collections +import dataclasses +import logging +from collections.abc import Callable, Generator, Iterable +from datetime import timedelta +from typing import TYPE_CHECKING, Any + +from nexusrpc import OutputT + +import temporalio.api.enums.v1 +import temporalio.api.failure.v1 +import temporalio.api.stream.v1 +import temporalio.api.streamservice.v1 +import temporalio.bridge.proto.workflow_commands +import temporalio.bridge.streams +import temporalio.exceptions +import temporalio.runtime +import temporalio.workflow +from temporalio.client._stream_store import ( + _STREAM_OPERATIONS, + StreamConfig, + StreamReplaySource, + StreamStore, + _request_ref, +) +from temporalio.converter import StorageDriverWorkflowInfo + +from ._interceptor import StartNexusOperationInput + +if TYPE_CHECKING: + from ._workflow_instance import _WorkflowInstanceImpl + +logger = logging.getLogger(__name__) + + +def _no_stream_config_error() -> temporalio.exceptions.ApplicationError: # pyright: ignore[reportUnusedFunction] + # A `StreamConfigError` would fail the Workflow Task, and its retries would fail the + # same way for good. A Worker with no config is a setup that a user has to see, so + # the Workflow fails instead. + return temporalio.exceptions.ApplicationError( + "The client of this Worker has no Stream Store. Give one in " + "stream_config=StreamConfig(stores=[...]).", + type="StreamConfigError", + non_retryable=True, + ) + + +# The word of an error message for each write that Core makes for Workflow code, by Nexus +# operation name. +_DIRECT_WRITES = {"AppendRecords": "append", "CloseStream": "close"} + + +def _not_a_workflow_producer_error( + operation: str, +) -> temporalio.exceptions.StreamError: + # Core sets the Workflow Task as the epoch of the Producer, which only a Producer of + # kind Workflow has. + call = _DIRECT_WRITES[operation] + return temporalio.exceptions.StreamError( + f"A Stream {call} of Workflow code writes as a Producer of kind Workflow. " + "Leave out producer=, or give a Producer of kind Workflow.", + reason=temporalio.api.enums.v1.StreamErrorReason.STREAM_ERROR_REASON_INVALID, + ) + + +_STREAM_ERROR_DETAILS_TYPE = ( + temporalio.api.stream.v1.StreamErrorDetails.DESCRIPTOR.full_name +) + + +def _stream_error_from_core( # pyright: ignore[reportUnusedFunction] + failure: temporalio.api.failure.v1.Failure, +) -> temporalio.exceptions.StreamError | None: + """Give the Store error of a failure that Core made, or ``None`` for another one.""" + # Core gives a Store error the token of its reason as its type, and puts the reason in + # its details with no codec. The Local Activity path gives the Store's message alone, + # so the token comes off the message. + info = failure.application_failure_info + for payload in info.details.payloads: + if payload.metadata.get("messageType") != _STREAM_ERROR_DETAILS_TYPE.encode(): + continue + details = temporalio.api.stream.v1.StreamErrorDetails.FromString(payload.data) + message = failure.message.removeprefix(f"{info.type}: ") + return temporalio.exceptions._stream_error_from_reason(details.reason, message) + return None + + +def _start_direct_append( + instance: _WorkflowInstanceImpl, + input: StartNexusOperationInput[Any, OutputT], + request: temporalio.api.streamservice.v1.AppendRecordsRequest, + store_name: str, +) -> temporalio.workflow.NexusOperationHandle[OutputT]: + """Send an append of a Workflow Producer to Core, which makes it in the Workflow Task.""" + payload_converter, _ = instance._system_nexus_converters(input) + append = temporalio.bridge.proto.workflow_commands.AppendToStream( + stream=temporalio.api.stream.v1.StreamRef(id=request.ref.id, store=store_name), + bodies=[record.body for record in request.records], + producer=request.producer, + sequence=request.sequence, + ) + append.producer.workflow.ClearField("epoch") + appended = instance._start_stream_append(append) + + async def result() -> OutputT: + response = await appended + return payload_converter._from_transfer_value( + temporalio.api.streamservice.v1.AppendRecordsResponse( + first_offset=response.first_offset, + last_offset=response.last_offset, + ), + input.output_type, + ) + + return _StreamStoreOperationHandle( + instance.create_task(result(), name=f"stream store: {input.operation_name}") + ) + + +def _start_direct_close( + instance: _WorkflowInstanceImpl, + input: StartNexusOperationInput[Any, OutputT], + request: temporalio.api.streamservice.v1.CloseStreamRequest, + store_name: str, +) -> temporalio.workflow.NexusOperationHandle[OutputT]: + """Send a close of a Workflow Producer to Core. + + Core records the intent in this Workflow Task, closes the Stream after the server took + the task, and gives the result in a later task. So a failed task closes nothing. + """ + payload_converter, _ = instance._system_nexus_converters(input) + close = temporalio.bridge.proto.workflow_commands.CloseStream( + stream=temporalio.api.stream.v1.StreamRef(id=request.ref.id, store=store_name), + producer=request.producer, + ) + close.producer.workflow.ClearField("epoch") + closed = instance._start_stream_close(close) + + async def result() -> OutputT: + await closed + return payload_converter._from_transfer_value( + temporalio.api.streamservice.v1.CloseStreamResponse(), input.output_type + ) + + return _StreamStoreOperationHandle( + instance.create_task(result(), name=f"stream store: {input.operation_name}") + ) + + +def _start_direct_call( + instance: _WorkflowInstanceImpl, + input: StartNexusOperationInput[Any, OutputT], + request: temporalio.api.streamservice.v1.AppendRecordsRequest + | temporalio.api.streamservice.v1.CloseStreamRequest, + store_name: str, +) -> temporalio.workflow.NexusOperationHandle[OutputT]: + """Send an append or a close of Workflow code to Core. + + Core makes these calls itself, with the Workflow Task as the epoch of the Producer. + + Raises: + temporalio.exceptions.StreamError: The Producer is not of kind Workflow. + """ + if request.producer.WhichOneof("kind") != "workflow": + raise _not_a_workflow_producer_error(input.operation_name) + if isinstance(request, temporalio.api.streamservice.v1.AppendRecordsRequest): + return _start_direct_append(instance, input, request, store_name) + return _start_direct_close(instance, input, request, store_name) + + +def _start_replayed_direct_call( # pyright: ignore[reportUnusedFunction] + instance: _WorkflowInstanceImpl, + input: StartNexusOperationInput[Any, OutputT], +) -> temporalio.workflow.NexusOperationHandle[OutputT] | None: + """Replay a Direct call on a Worker with no Stream config, or give ``None``. + + The marker of an append or a close holds its result, and the marker of a read can hold + its Records, so the replay needs no Store. A ref with no Store sends an empty name, and + Core takes the Store of History. + """ + if input.operation_name not in ("AppendRecords", "CloseStream", "ReadRecords"): + return None + payload_converter, _ = instance._system_nexus_converters(input) + request = payload_converter._to_transfer_value(input.input) + if isinstance(request, temporalio.api.streamservice.v1.ReadRecordsRequest): + # A replay of a read whose marker holds its Records reads no Store. + return _start_direct_read( + instance, input, request, request.ref.store, records_in_history=True + ) + if isinstance( + request, + ( + temporalio.api.streamservice.v1.AppendRecordsRequest, + temporalio.api.streamservice.v1.CloseStreamRequest, + ), + ): + return _start_direct_call(instance, input, request, request.ref.store) + return None + + +# A client's read gives at most this many Records in a page, and a `max_records` of zero +# or less gives this many. A read of Workflow code takes the same limits. +_MAX_READ_RECORDS = 1000 +# The longest `wait` of a client's read. A longer one waits this long. +_MAX_READ_WAIT = timedelta(seconds=60) + + +@dataclasses.dataclass(frozen=True) +class _ReadKey: + """What a ``read_page`` call of Workflow code must match to take an open read.""" + + store: str + stream_id: str + offset: bytes + """The offset that the next page starts after. Empty is the start of the Stream.""" + wait: timedelta | None + + +class _DirectRead: # pyright: ignore[reportUnusedClass] + """One Core read of a Stream, which serves the ``read_page`` calls of Workflow code. + + Core gives its Records in batches. Each ``read_page`` call takes one page of them, and the + read then waits for the next call under the offset of that page. + """ + + def __init__(self, seq: int) -> None: + self.seq = seq + self.records: collections.deque[temporalio.api.stream.v1.StreamRecord] = ( + collections.deque() + ) + # Whether Core still reads, with no end and no close. + self.running = True + self.done = False + self.error: BaseException | None = None + # A job and an end wake the call that waits, which is at most one. + self._waiters: list[asyncio.Future[None]] = [] + + def kept(self) -> bool: + """Whether the read can serve a later call.""" + # A failure that no call raised yet stays, so the next call raises it. + return self.running or bool(self.records) or self.error is not None + + def take( + self, + records: Iterable[temporalio.api.stream.v1.StreamRecord], + *, + end: bool, + done: bool, + error: BaseException | None, + ) -> None: + """Keep the Records and the end of one job of Core.""" + self.records.extend(records) + if end: + self.running = False + self.done = done + self.error = error + self._wake() + + def close(self) -> None: + """Mark the read as closed in Core. Its buffered Records stay.""" + self.running = False + self._wake() + + def _wake(self) -> None: + for waiter in self._waiters: + if not waiter.done(): + waiter.set_result(None) + + async def page( + self, max_records: int, max_bytes: int | None + ) -> tuple[list[temporalio.api.stream.v1.StreamRecord], bool]: + """Give the next page and whether the Stream is done, and wait when none is buffered. + + Raises: + temporalio.exceptions.StreamError: The read failed. + """ + while not self.records and self.running: + waiter: asyncio.Future[None] = asyncio.get_running_loop().create_future() + self._waiters.append(waiter) + try: + await waiter + finally: + self._waiters.remove(waiter) + if not self.records and self.error is not None: + error, self.error = self.error, None + raise error + page: list[temporalio.api.stream.v1.StreamRecord] = [] + size = 0 + while self.records and len(page) < max_records: + record_size = self.records[0].ByteSize() + if page and max_bytes is not None and size + record_size > max_bytes: + break + page.append(self.records.popleft()) + size += record_size + return page, self.done and not self.records + + +def _read_limits( + request: temporalio.api.streamservice.v1.ReadRecordsRequest, +) -> tuple[int, timedelta | None, int | None]: + """Give the ``max_records``, ``wait`` and ``max_bytes`` of a client's read. + + Raises: + temporalio.exceptions.StreamError: ``max_bytes`` is negative. + """ + max_records = request.max_records + if max_records <= 0 or max_records > _MAX_READ_RECORDS: + max_records = _MAX_READ_RECORDS + wait: timedelta | None = None + if request.HasField("wait") and request.wait.ToTimedelta() > timedelta(): + wait = min(request.wait.ToTimedelta(), _MAX_READ_WAIT) + if request.max_bytes < 0: + raise temporalio.exceptions.StreamError( + "max_bytes must be 0 or more", + reason=temporalio.api.enums.v1.StreamErrorReason.STREAM_ERROR_REASON_INVALID, + ) + return max_records, wait, request.max_bytes or None + + +def _start_direct_read( + instance: _WorkflowInstanceImpl, + input: StartNexusOperationInput[Any, OutputT], + request: temporalio.api.streamservice.v1.ReadRecordsRequest, + store_name: str, + *, + records_in_history: bool, +) -> temporalio.workflow.NexusOperationHandle[OutputT]: + """Answer one ``read_page`` call of Workflow code from a Core read of the Stream.""" + instance._assert_not_read_only("start stream read") + payload_converter, _ = instance._system_nexus_converters(input) + max_records, wait, max_bytes = _read_limits(request) + key = _ReadKey( + store=store_name, + stream_id=request.ref.id, + offset=request.from_offset, + wait=wait, + ) + read = instance._take_stream_read(key) + if read is None: + start = temporalio.bridge.proto.workflow_commands.StartStreamRead( + stream=temporalio.api.stream.v1.StreamRef( + id=request.ref.id, store=store_name + ), + from_offset=request.from_offset, + records_in_history=records_in_history, + ) + # The pages come from the calls, so the Core read has no cap and no timeout. + # With no `wait`, an idle timeout of zero gives what is in the Stream. + start.idle_timeout.SetInParent() + start.idle_timeout.FromTimedelta(wait or timedelta()) + read = instance._start_stream_read(start) + + async def result() -> OutputT: + try: + records, done = await read.page(max_records, max_bytes) + except BaseException: + instance._park_stream_read(read, key) + raise + last_offset = records[-1].offset if records else request.from_offset + instance._park_stream_read(read, dataclasses.replace(key, offset=last_offset)) + return payload_converter._from_transfer_value( + temporalio.api.streamservice.v1.ReadRecordsResponse( + records=records, last_offset=last_offset, done=done + ), + input.output_type, + ) + + return _StreamStoreOperationHandle( + instance.create_task(result(), name=f"stream store: {input.operation_name}") + ) + + +def _bridge_read_ahead_bytes(config: StreamConfig | None) -> int | None: # pyright: ignore[reportUnusedFunction] + """Give Core the read-ahead bound of the config, or ``None`` for Core's default.""" + return config.workflow_read_ahead_bytes if config is not None else None + + +def _bridge_read_page_records(config: StreamConfig | None) -> int | None: # pyright: ignore[reportUnusedFunction] + """Give Core the page cap of a read in the config, or ``None`` for Core's default.""" + return config.workflow_read_page_records if config is not None else None + + +def _bridge_read_min_gap(config: StreamConfig | None) -> timedelta | None: # pyright: ignore[reportUnusedFunction] + """Give Core the least gap between two batches of a read, or ``None`` for no gap.""" + return config.workflow_read_min_gap if config is not None else None + + +# A Store that raises something other than a `StreamError` with a reason has a fault that a +# retry reads again, as with corrupt data, so Core gives the failure on the first try. +_STORE_FAULT = ( + temporalio.api.enums.v1.StreamErrorReason.STREAM_ERROR_REASON_STORAGE_CORRUPT +) + + +class _CoreStoreCaller: + """Runs the calls that Core makes on a Store written in Python. + + Core calls :py:meth:`start` from a thread of the bridge. Each call runs as a task on + the Worker's event loop, as a call of client or Activity code does, and never on a + Workflow thread. The bridge cancels the task when Core stops the call. + """ + + def __init__(self, store: StreamStore) -> None: + self._store = store + self._loop: asyncio.AbstractEventLoop | None = None + + def bind(self, loop: asyncio.AbstractEventLoop) -> None: + """Run the later calls on ``loop``, the event loop of the Worker.""" + self._loop = loop + + def start( + self, + operation: str, + request: bytes, + done: Callable[[bytes | None, int, str], None], + ) -> Callable[[], None]: + """Start one call on the event loop, and give what cancels it. + + ``done`` gets the encoded response, or ``None`` with the reason and the message + of the failure. + """ + loop = self._loop + if loop is None: + raise RuntimeError("The Worker of the Stream Store isn't running") + call = self._call(operation, request, done) + try: + future = asyncio.run_coroutine_threadsafe(call, loop) + except BaseException: + call.close() + raise + + def cancel() -> None: + # A closed loop has no task left, and its cancel would raise. + if not loop.is_closed(): + future.cancel() + + return cancel + + async def _call( + self, + operation: str, + request: bytes, + done: Callable[[bytes | None, int, str], None], + ) -> None: + stream_operation = _STREAM_OPERATIONS[operation] + name = self._store.name + try: + response = await stream_operation.call( + self._store, stream_operation.request_type.FromString(request) + ) + except temporalio.exceptions.StreamError as err: + if ( + err.reason + != temporalio.api.enums.v1.StreamErrorReason.STREAM_ERROR_REASON_UNSPECIFIED + ): + done(None, err.reason, err.message) + return + done( + None, + _STORE_FAULT, + f"The Stream Store '{name}' raised a StreamError with no reason: " + f"{err.message}", + ) + except Exception as err: + # Workflow code gets only the message, so the Worker logs the traceback. + logger.warning( + "The Stream Store '%s' raised in %s", name, operation, exc_info=True + ) + done( + None, + _STORE_FAULT, + f"The Stream Store '{name}' raised {type(err).__name__}: {err}", + ) + else: + done(response.SerializeToString(), 0, "") + + +@dataclasses.dataclass(frozen=True) +class _CoreStreamRuntimes: + """The Stream runtimes that a Worker gives Core, and the callers of its Python Stores.""" + + runtimes: list[tuple[str, Any]] + """The bridge's object of each Store's Stream runtime, by Store name.""" + + callers: list[_CoreStoreCaller] + + def bind(self, loop: asyncio.AbstractEventLoop) -> None: + """Run the calls of each Store written in Python on ``loop``.""" + for caller in self.callers: + caller.bind(loop) + + +def _core_stream_runtimes( # pyright: ignore[reportUnusedFunction] + config: StreamConfig | None, runtime: temporalio.runtime.Runtime +) -> _CoreStreamRuntimes: + """Give Core a Stream runtime for each Store of the config. + + Each runtime is on ``runtime`` and calls its Store on the event loop that + :py:meth:`_CoreStreamRuntimes.bind` gives. + """ + if config is None: + return _CoreStreamRuntimes(runtimes=[], callers=[]) + runtimes: list[tuple[str, Any]] = [] + callers: list[_CoreStoreCaller] = [] + for store in config.stores: + caller = _CoreStoreCaller(store) + stream_runtime = temporalio.bridge.streams.StreamRuntime.new_python( + runtime._core_runtime, caller=caller, limits=store.limits + ) + runtimes.append((store.name, stream_runtime._ref)) + callers.append(caller) + return _CoreStreamRuntimes(runtimes=runtimes, callers=callers) + + +def _workflow_target() -> StorageDriverWorkflowInfo: + # External storage builds the same value for a Workflow. + info = temporalio.workflow.info() + return StorageDriverWorkflowInfo( + id=info.workflow_id, + type=info.workflow_type, + run_id=info.run_id, + namespace=info.namespace, + ) + + +@dataclasses.dataclass(frozen=True) +class _WorkflowStreamStore: + """The Worker's Stream Stores as Workflow code reaches them.""" + + config: StreamConfig + """The Stream config of the Worker's client.""" + + @classmethod + def from_config(cls, config: StreamConfig | None) -> _WorkflowStreamStore | None: + """Build it from the Stream config of a client, or give ``None`` when it has none.""" + return cls(config=config) if config is not None else None + + def start_operation( + self, + instance: _WorkflowInstanceImpl, + input: StartNexusOperationInput[Any, OutputT], + ) -> temporalio.workflow.NexusOperationHandle[OutputT]: + """Answer a Stream call of Workflow code with the Store that the call goes to.""" + stream_operation = _STREAM_OPERATIONS[input.operation_name] + payload_converter, _ = instance._system_nexus_converters(input) + request = payload_converter._to_transfer_value(input.input) + ref = _request_ref(input.input) + # The choice runs in Workflow code, so a ref of a Store that the config doesn't + # have raises `StreamConfigError` here, with no I/O, and fails the Workflow Task. + store = self.config._store_for(ref, _workflow_target()) + options = self.config.workflow_call_options + if isinstance(request, temporalio.api.streamservice.v1.ReadRecordsRequest): + instance._assert_not_read_only("start stream read") + return _start_direct_read( + instance, + input, + request, + store.name, + records_in_history=( + self.config.workflow_read_replay_from is StreamReplaySource.HISTORY + ), + ) + if isinstance( + request, + ( + temporalio.api.streamservice.v1.AppendRecordsRequest, + temporalio.api.streamservice.v1.CloseStreamRequest, + ), + ): + return _start_direct_call(instance, input, request, store.name) + + # A create and a describe run in the Store's Local Activity. + async def result() -> OutputT: + try: + response = await store.call_from_workflow( + input.operation_name, request, options + ) + except temporalio.exceptions.ApplicationError as err: + # Core hands back a failed Local Activity's error without a wrapper. + stream_error = temporalio.exceptions._stream_error_from_failure(err) + if stream_error is not None: + raise stream_error from None + raise + except temporalio.exceptions.ActivityError as err: + # Workflow code gets the Store's error, not the Local Activity's wrapper. + if err.cause is not None: + stream_error = temporalio.exceptions._stream_error_from_failure( + err.cause + ) + if stream_error is not None: + raise stream_error from None + if isinstance( + err.cause, + ( + temporalio.exceptions.ApplicationError, + temporalio.exceptions.TimeoutError, + ), + ): + raise err.cause + raise + return payload_converter._from_transfer_value( + stream_operation.check_response(store, response), input.output_type + ) + + return _StreamStoreOperationHandle( + instance.create_task(result(), name=f"stream store: {input.operation_name}") + ) + + +class _StreamStoreOperationHandle(temporalio.workflow.NexusOperationHandle[OutputT]): + def __init__(self, task: asyncio.Task[OutputT]) -> None: + self._task = task + + def cancel(self) -> bool: + return self._task.cancel() + + def __await__(self) -> Generator[Any, Any, OutputT]: + return self._task.__await__() + + @property + def operation_token(self) -> str | None: + return None diff --git a/temporalio/worker/_worker.py b/temporalio/worker/_worker.py index e06bb99a4b..4983f9a0f0 100644 --- a/temporalio/worker/_worker.py +++ b/temporalio/worker/_worker.py @@ -34,6 +34,13 @@ from ._interceptor import Interceptor from ._nexus import _NexusWorker from ._plugin import Plugin +from ._stream_store import ( + _bridge_read_ahead_bytes, + _bridge_read_min_gap, + _bridge_read_page_records, + _core_stream_runtimes, + _WorkflowStreamStore, +) from ._tuning import WorkerTuner from ._workflow import ( _DEFAULT_WORKFLOW_TASK_EXTERNAL_STORAGE_CONCURRENCY, @@ -498,7 +505,11 @@ def _init_from_config(self, client: temporalio.client.Client, config: WorkerConf or bridge_client.config.runtime or temporalio.runtime.Runtime.default() ) - activities = config.get("activities") + activities = list(config.get("activities") or []) + stream_config = client_config.get("stream_config") + if stream_config is not None and config.get("workflows"): + for store in stream_config.stores: + activities.extend(store.worker_activities()) if activities: _warn_if_activity_executor_max_workers_is_inconsistent(config) self._activity_worker = _ActivityWorker( @@ -578,6 +589,7 @@ def check_activity(activity: str): encode_headers=client_config["header_codec_behavior"] != HeaderCodecBehavior.NO_CODEC, max_workflow_task_external_storage_concurrency=max_workflow_task_external_storage_concurrency, + stream_store=_WorkflowStreamStore.from_config(stream_config), ) tuner = config.get("tuner") @@ -642,6 +654,13 @@ def check_activity(activity: str): driver.type() for driver in self._storage_drivers } + # Only Workflow code makes Direct calls, so a Worker with no Workflows builds no + # runtime, as it builds no Store Activity. + self._core_stream_runtimes = _core_stream_runtimes( + stream_config if self._workflow_worker is not None else None, + self._runtime, + ) + # Create bridge worker last. We have empirically observed that if it is # created before an error is raised from the activity worker # constructor, a deadlock/hang will occur presumably while trying to @@ -667,7 +686,9 @@ def check_activity(activity: str): enable_workflows=self._workflow_worker is not None, enable_local_activities=self._activity_worker is not None and self._workflow_worker is not None, - enable_remote_activities=self._activity_worker is not None + # The Activities of the Stream Stores run as Local Activities, so + # alone they don't make this worker poll for Activity Tasks. + enable_remote_activities=bool(config.get("activities")) and not config.get("no_remote_activities"), enable_nexus=self._nexus_worker is not None, ), @@ -701,6 +722,13 @@ def check_activity(activity: str): if self._workflow_worker else set() ), + stream_replay_as_workflow_fail=self._workflow_worker is not None + and self._workflow_worker.stream_replay_as_workflow_fail(), + stream_replay_as_workflow_fail_for_types=( + self._workflow_worker.stream_replay_as_workflow_fail_for_types() + if self._workflow_worker + else set() + ), versioning_strategy=versioning_strategy, workflow_task_poller_behavior=_poller_behavior_to_bridge( workflow_task_poller_behavior @@ -716,6 +744,10 @@ def check_activity(activity: str): disable_payload_error_limit=config.get( "disable_payload_error_limit", False ), + stream_runtimes=self._core_stream_runtimes.runtimes, + stream_read_ahead_bytes=_bridge_read_ahead_bytes(stream_config), + stream_read_page_records=_bridge_read_page_records(stream_config), + stream_read_min_gap=_bridge_read_min_gap(stream_config), ), ) @@ -751,13 +783,19 @@ def client(self, value: temporalio.client.Client) -> None: Changing the client will make sure the worker starts using it for the next calls it makes. However, outstanding client calls will still complete with the existing client. The new client cannot be "lazy" and - must be using the same runtime as the current client. + must be using the same runtime and Stream Stores as the current client. """ bridge_client = _extract_bridge_client_for_worker(value) if self._runtime is not bridge_client._runtime: raise ValueError( "New client is not on the same runtime as the existing client" ) + # Running Workflows route their Stream calls by the Store that the worker + # started with, so a new client can't change it. + if _stream_config_stores(value) != _stream_config_stores(self.client): + raise ValueError( + "New client doesn't have the Stream Stores of the existing client" + ) assert bridge_client._bridge_client self._bridge_worker.replace_client(bridge_client._bridge_client) self._config["client"] = value @@ -832,6 +870,8 @@ async def _run(self): if self._started: raise RuntimeError("Already started") self._started = True + # Core calls a Store written in Python on the loop that runs the Worker. + self._core_stream_runtimes.bind(asyncio.get_running_loop()) # Eagerly validate which will do a namespace check in Core. try: @@ -1013,6 +1053,13 @@ async def __aexit__(self, exc_type: type[BaseException] | None, *args: Any) -> N raise self._async_context_run_exception +def _stream_config_stores( + client: temporalio.client.Client, +) -> list[temporalio.client.StreamStore]: + config = client.config(active_config=True).get("stream_config") + return list(config.stores) if config is not None else [] + + class WorkerConfig(TypedDict, total=False): """TypedDict of config originally passed to :py:class:`Worker`.""" diff --git a/temporalio/worker/_workflow.py b/temporalio/worker/_workflow.py index d8380a6b90..74e40dfd18 100644 --- a/temporalio/worker/_workflow.py +++ b/temporalio/worker/_workflow.py @@ -40,6 +40,7 @@ WorkflowInboundInterceptor, WorkflowInterceptorClassInput, ) +from ._stream_store import _WorkflowStreamStore from ._workflow_instance import ( _DEFAULT_ENABLED_WORKFLOW_LOGIC_FLAGS, PatchActivationInput, @@ -128,6 +129,7 @@ def __init__( encode_headers: bool, max_workflow_task_external_storage_concurrency: int, default_workflow_logic_flags: frozenset[_WorkflowLogicFlag] | None = None, + stream_store: _WorkflowStreamStore | None = None, ) -> None: # Debug mode is enabled if specified or if the TEMPORAL_DEBUG env var is truthy debug_mode = debug_mode or bool(os.environ.get("TEMPORAL_DEBUG")) @@ -197,6 +199,7 @@ def __init__( self._max_workflow_task_external_storage_concurrency = ( max_workflow_task_external_storage_concurrency ) + self._stream_store = stream_store self._throw_after_activation: Exception | None = None # If debug mode is enabled, disable deadlock detection @@ -819,6 +822,7 @@ def _create_workflow_instance( nexus_serialization_context=temporalio.converter._serialization_context._propagated_nexus_serialization_context( init ), + stream_store=self._stream_store, ) if defn.sandboxed: return self._workflow_runner.create_instance(det) @@ -831,6 +835,12 @@ def nondeterminism_as_workflow_fail(self) -> bool: for typ in self._workflow_failure_exception_types ) + def stream_replay_as_workflow_fail(self) -> bool: + return any( + issubclass(temporalio.exceptions.StreamReplayError, typ) + for typ in self._workflow_failure_exception_types + ) + def _set_default_workflow_logic_flag( self, flag: _WorkflowLogicFlag, *, enabled: bool ) -> None: @@ -849,6 +859,16 @@ def nondeterminism_as_workflow_fail_for_types(self) -> set[str]: ) } + def stream_replay_as_workflow_fail_for_types(self) -> set[str]: + return { + k + for k, v in self._workflows.items() + if any( + issubclass(temporalio.exceptions.StreamReplayError, typ) + for typ in v.failure_exception_types + ) + } + class _DeadlockError(Exception): """Exception class for deadlocks. Contains functionality to swap the default traceback for another.""" diff --git a/temporalio/worker/_workflow_instance.py b/temporalio/worker/_workflow_instance.py index b4629a293a..dd159982ad 100644 --- a/temporalio/worker/_workflow_instance.py +++ b/temporalio/worker/_workflow_instance.py @@ -59,10 +59,7 @@ import temporalio.exceptions import temporalio.nexus.system import temporalio.workflow -from temporalio.client._stream_store import ( - _STREAM_SERVICE, - _workflow_stream_call_error, -) +from temporalio.client._stream_store import _STREAM_SERVICE from temporalio.converter import StorageDriverStoreContext, StorageDriverWorkflowInfo from temporalio.nexus.system.stream_service.workflow._system_nexus_interceptor import ( _start_system_nexus_operation as _start_stream_system_nexus_operation, @@ -98,6 +95,14 @@ WorkflowInboundInterceptor, WorkflowOutboundInterceptor, ) +from ._stream_store import ( + _DirectRead, + _no_stream_config_error, + _ReadKey, + _start_replayed_direct_call, + _stream_error_from_core, + _WorkflowStreamStore, +) logger = logging.getLogger(__name__) @@ -195,6 +200,10 @@ class WorkflowInstanceDetails: temporalio.converter.NexusSerializationContext | None ) = None """Serialization context of the Nexus operation handler that started the workflow, if any.""" + stream_store: _WorkflowStreamStore | None = None + """The Worker's Stream Stores as Workflow code reaches them, or ``None`` when the + Worker's client has none. This release has no Streams on the server, so with ``None`` + a Stream call fails the Workflow.""" class WorkflowInstance(ABC): @@ -264,6 +273,17 @@ def get_thread_id(self) -> int | None: return None +def _is_completion( + command: temporalio.bridge.proto.workflow_commands.workflow_commands_pb2.WorkflowCommand, +) -> bool: + return ( + command.HasField("complete_workflow_execution") + or command.HasField("continue_as_new_workflow_execution") + or command.HasField("fail_workflow_execution") + or command.HasField("cancel_workflow_execution") + ) + + class UnsandboxedWorkflowRunner(WorkflowRunner): """Workflow runner that does not do any sandboxing.""" @@ -311,6 +331,7 @@ def __init__(self, det: WorkflowInstanceDetails) -> None: # the Nexus caller, which decodes them with the handler's Nexus context. A retried or cron run # reads the previous run's result and failure with the same context. self._nexus_serialization_context = det.nexus_serialization_context + self._stream_store = det.stream_store if self._nexus_serialization_context: self._result_payload_converter = self._payload_converter_with_context( self._nexus_serialization_context @@ -358,6 +379,19 @@ def __init__(self, det: WorkflowInstanceDetails) -> None: self._pending_nexus_operations: dict[int, _NexusOperationHandle] = {} self._pending_external_signals: dict[int, tuple[asyncio.Future, str]] = {} self._pending_external_cancels: dict[int, tuple[asyncio.Future, str]] = {} + # Core reads that run, by seq. + self._pending_stream_reads: dict[int, _DirectRead] = {} + # Reads that no call waits on, by what the next call must match. Each holds the + # Core read and the Records that it gave. + self._parked_stream_reads: dict[_ReadKey, list[_DirectRead]] = {} + self._pending_stream_appends: dict[ + int, + asyncio.Future[temporalio.bridge.proto.workflow_activation.StreamAppended], + ] = {} + self._pending_stream_closes: dict[int, asyncio.Future[None]] = {} + # Reads that the SDK closed. A job of one of them can still come in the activation + # that the close answers, and is dropped. + self._closed_stream_reads: set[int] = set() # Keyed by type self._curr_seqs: dict[str, int] = {} # TODO(cretz): Any concerns about not sharing this? Maybe the types I @@ -571,6 +605,17 @@ def activate( self._run_once( check_conditions=index == 1 or index == 2, jobs=act.jobs ) + + # A query-only activation sends no command, and the end of the Workflow + # stops every read in Core. + if ( + not job_sets[3] + and not self._deleting + and not any( + map(_is_completion, self._current_completion.successful.commands) + ) + ): + self._close_parked_stream_reads() except Exception as err: # We want some errors during activation, like those that can happen # during payload conversion, to be able to fail the workflow not the @@ -635,17 +680,7 @@ def activate( ) self._current_completion.failed.failure.application_failure_info.SetInParent() - def is_completion( - command: temporalio.bridge.proto.workflow_commands.workflow_commands_pb2.WorkflowCommand, - ): - return ( - command.HasField("complete_workflow_execution") - or command.HasField("continue_as_new_workflow_execution") - or command.HasField("fail_workflow_execution") - or command.HasField("cancel_workflow_execution") - ) - - if any(map(is_completion, self._current_completion.successful.commands)): + if any(map(_is_completion, self._current_completion.successful.commands)): self._warn_if_unfinished_handlers() return self._current_completion @@ -687,6 +722,12 @@ def _apply( self._apply_resolve_signal_external_workflow( job.resolve_signal_external_workflow ) + elif job.HasField("resolve_stream_read"): + self._apply_resolve_stream_read(job.resolve_stream_read) + elif job.HasField("resolve_stream_append"): + self._apply_resolve_stream_append(job.resolve_stream_append) + elif job.HasField("resolve_stream_close"): + self._apply_resolve_stream_close(job.resolve_stream_close) elif job.HasField("signal_workflow"): self._apply_signal_workflow(job.signal_workflow) elif job.HasField("initialize_workflow"): @@ -938,6 +979,64 @@ def _apply_remove_from_cache( for task in self._tasks: task.cancel() + def _apply_resolve_stream_read( + self, job: temporalio.bridge.proto.workflow_activation.ResolveStreamRead + ) -> None: + read = self._pending_stream_reads.get(job.seq) + if read is None: + if job.seq in self._closed_stream_reads: + return + raise RuntimeError(f"Failed finding Stream read for sequence {job.seq}") + error: BaseException | None = None + end = job.HasField("end") + if end: + del self._pending_stream_reads[job.seq] + if job.end.HasField("failure"): + error = _stream_error_from_core(job.end.failure) + if error is None: + error = self._context_free_failure_converter.from_failure( + job.end.failure, self._context_free_payload_converter + ) + read.take(job.records, end=end, done=job.end.done, error=error) + + def _apply_resolve_stream_append( + self, job: temporalio.bridge.proto.workflow_activation.ResolveStreamAppend + ) -> None: + fut = self._pending_stream_appends.pop(job.seq, None) + if fut is None: + raise RuntimeError(f"Failed finding Stream append for sequence {job.seq}") + # Code that stopped waiting for the append has cancelled the future. + if fut.done(): + return + if job.HasField("appended"): + fut.set_result(job.appended) + return + error: BaseException | None = _stream_error_from_core(job.failed) + if error is None: + error = self._context_free_failure_converter.from_failure( + job.failed, self._context_free_payload_converter + ) + fut.set_exception(error) + + def _apply_resolve_stream_close( + self, job: temporalio.bridge.proto.workflow_activation.ResolveStreamClose + ) -> None: + fut = self._pending_stream_closes.pop(job.seq, None) + if fut is None: + raise RuntimeError(f"Failed finding Stream close for sequence {job.seq}") + # Code that stopped waiting for the close has cancelled the future. + if fut.done(): + return + if not job.HasField("failed"): + fut.set_result(None) + return + error: BaseException | None = _stream_error_from_core(job.failed) + if error is None: + error = self._context_free_failure_converter.from_failure( + job.failed, self._context_free_payload_converter + ) + fut.set_exception(error) + def _apply_resolve_activity( self, job: temporalio.bridge.proto.workflow_activation.ResolveActivity ) -> None: @@ -1770,6 +1869,66 @@ async def workflow_start_child_workflow( ) ) + def _start_stream_read( + self, start: temporalio.bridge.proto.workflow_commands.StartStreamRead + ) -> _DirectRead: + start.seq = self._next_seq("stream_read") + self._add_command().start_stream_read.CopyFrom(start) + read = _DirectRead(start.seq) + self._pending_stream_reads[start.seq] = read + return read + + def _take_stream_read(self, key: _ReadKey) -> _DirectRead | None: + reads = self._parked_stream_reads.get(key) + if not reads: + return None + read = reads.pop(0) + if not reads: + del self._parked_stream_reads[key] + return read + + def _park_stream_read(self, read: _DirectRead, key: _ReadKey) -> None: + if read.kept(): + self._parked_stream_reads.setdefault(key, []).append(read) + + def _close_parked_stream_reads(self) -> None: + # A Core read holds its Workflow Task open, so one that no call waits on closes + # before the activation completes. Its Records stay for a later call. + for key, reads in list(self._parked_stream_reads.items()): + for read in reads: + if read.running: + self._add_command().close_stream_read.seq = read.seq + self._closed_stream_reads.add(read.seq) + del self._pending_stream_reads[read.seq] + read.close() + kept = [read for read in reads if read.kept()] + if kept: + self._parked_stream_reads[key] = kept + else: + del self._parked_stream_reads[key] + + def _start_stream_append( + self, append: temporalio.bridge.proto.workflow_commands.AppendToStream + ) -> asyncio.Future[temporalio.bridge.proto.workflow_activation.StreamAppended]: + command = self._add_command() + append.seq = self._next_seq("stream_append") + command.append_to_stream.CopyFrom(append) + fut: asyncio.Future[ + temporalio.bridge.proto.workflow_activation.StreamAppended + ] = self.create_future() + self._pending_stream_appends[append.seq] = fut + return fut + + def _start_stream_close( + self, close: temporalio.bridge.proto.workflow_commands.CloseStream + ) -> asyncio.Future[None]: + command = self._add_command() + close.seq = self._next_seq("stream_close") + command.close_stream.CopyFrom(close) + fut: asyncio.Future[None] = self.create_future() + self._pending_stream_closes[close.seq] = fut + return fut + def workflow_start_local_activity( self, activity: Any, @@ -2312,25 +2471,7 @@ async def operation_handle_fn() -> OutputT: ) if temporalio.nexus.system.is_system_endpoint(input.endpoint): - serialization_context = temporalio.nexus.system._get_serialization_context( - input.service, - input.operation_name, - input.input, - ) - user_payload_converter = self._context_free_payload_converter - user_failure_converter = self._context_free_failure_converter - if serialization_context is not None: - user_payload_converter = self._payload_converter_with_context( - serialization_context - ) - user_failure_converter = self._failure_converter_with_context( - serialization_context - ) - payload_converter = temporalio.nexus.system._get_payload_converter( - user_payload_converter, - user_failure_converter, - ) - failure_converter = user_failure_converter + payload_converter, failure_converter = self._system_nexus_converters(input) else: serialization_context = temporalio.converter.NexusSerializationContext( endpoint=input.endpoint, @@ -2369,12 +2510,48 @@ async def _intercept_system_nexus_operation( async def _schedule_system_nexus_operation( self, input: StartNexusOperationInput[Any, OutputT] ) -> temporalio.workflow.NexusOperationHandle[OutputT]: - # The server has no Streams, and Workflow code reaches no Store. So a Stream call - # stops here, after the interceptors saw it. + # This release has no Streams on the server, so a Worker answers a Stream call with + # a Store of its client. Every other system Nexus operation goes to the server. if input.service == _STREAM_SERVICE.name: - raise _workflow_stream_call_error() + store = self._stream_store + if store is None: + replayed = ( + _start_replayed_direct_call(self, input) + if self._is_replaying + else None + ) + if replayed is None: + raise _no_stream_config_error() + return replayed + return store.start_operation(self, input) return await self._outbound_start_nexus_operation(input) + def _system_nexus_converters( + self, input: StartNexusOperationInput[Any, Any] + ) -> tuple[ + temporalio.nexus.system._SystemNexusPayloadConverter, + temporalio.converter.FailureConverter, + ]: + serialization_context = temporalio.nexus.system._get_serialization_context( + input.service, + input.operation_name, + input.input, + ) + user_payload_converter = self._context_free_payload_converter + user_failure_converter = self._context_free_failure_converter + if serialization_context is not None: + user_payload_converter = self._payload_converter_with_context( + serialization_context + ) + user_failure_converter = self._failure_converter_with_context( + serialization_context + ) + payload_converter = temporalio.nexus.system._get_payload_converter( + user_payload_converter, + user_failure_converter, + ) + return payload_converter, user_failure_converter + #### Miscellaneous helpers #### # These are in alphabetical order. @@ -2541,6 +2718,11 @@ def get_serialization_context( workflow_id=self._info.workflow_id, ) + if command_info.stream: + # Stream bodies take the codec with no context on every path, so a client, an + # Activity and a Workflow read what each other wrote. + return None + if self._nexus_serialization_context and command_info.command_type in ( temporalio.api.enums.v1.command_type_pb2.CommandType.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, temporalio.api.enums.v1.command_type_pb2.CommandType.COMMAND_TYPE_FAIL_WORKFLOW_EXECUTION, diff --git a/tests/contrib/streams/test_appender_workflow.py b/tests/contrib/streams/test_appender_workflow.py new file mode 100644 index 0000000000..f7cc6f1ce1 --- /dev/null +++ b/tests/contrib/streams/test_appender_workflow.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import uuid +from datetime import timedelta +from typing import Any + +from temporalio import workflow +from temporalio.client import Client, WorkflowHandle +from temporalio.contrib.streams import BufferedAppender +from temporalio.nexus.system.stream_service import StreamRef +from temporalio.nexus.system.stream_service.workflow import get_stream +from tests.helpers import new_worker +from tests.helpers.streams import DictStreamStore +from tests.nexus.test_stream_store import _with_store + + +@workflow.defn +class AppenderWorkflow: + @workflow.run + async def run(self, stream_id: str) -> str: + stream = get_stream(StreamRef(id=stream_id), body_type=str) + try: + BufferedAppender(stream) # type: ignore[arg-type] + except RuntimeError as err: + return str(err) + return "built" + + +async def _run( + client: Client, workflow_type: type[Any], stream_id: str +) -> tuple[WorkflowHandle[Any, Any], Any]: + async with new_worker(client, workflow_type) as worker: + handle: WorkflowHandle[Any, Any] = await client.start_workflow( + workflow_type.run, + stream_id, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + execution_timeout=timedelta(seconds=30), + ) + return handle, await handle.result() + + +async def test_workflow_code_cannot_build_an_appender(client: Client): + _, message = await _run( + _with_store(client, DictStreamStore()), + AppenderWorkflow, + f"stream-{uuid.uuid4()}", + ) + assert message == ( + "BufferedAppender is for client and Activity code. In Workflow code, pass the " + "Records to one append call." + ) diff --git a/tests/contrib/streams/test_producer.py b/tests/contrib/streams/test_producer.py index 86b6d922c9..0d6386da44 100644 --- a/tests/contrib/streams/test_producer.py +++ b/tests/contrib/streams/test_producer.py @@ -22,9 +22,11 @@ ProducerKindWorkflowActivity, StreamRecord, StreamRef, + StreamStatus, WorkflowProducer, ) from temporalio.nexus.system.stream_service.client import Stream +from temporalio.nexus.system.stream_service.workflow import create_stream from temporalio.testing import ActivityEnvironment from tests.helpers import new_worker from tests.helpers.streams import DictStreamStore @@ -122,25 +124,43 @@ async def append_and_give_producer() -> Producer: @workflow.defn class CurrentProducerWorkflow: @workflow.run - async def run(self) -> str: - try: - current_producer() - except RuntimeError as err: - return str(err) - return "no error" + async def run(self, stream_id: str) -> Producer: + stream = await create_stream(id=stream_id, body_type=str) + producer = current_producer() + await stream.append("a", producer=producer) + await stream.close(producer=producer) + return producer -async def test_current_producer_in_workflow_code_raises(client: Client): - async with new_worker(client, CurrentProducerWorkflow) as worker: - message = await client.execute_workflow( +async def test_current_producer_in_workflow_code_is_the_workflow_producer( + client: Client, +): + stream_client = _with_store(client, DictStreamStore()) + stream_id = f"stream-{uuid.uuid4()}" + async with new_worker(stream_client, CurrentProducerWorkflow) as worker: + handle = await stream_client.start_workflow( CurrentProducerWorkflow.run, + stream_id, id=f"workflow-{uuid.uuid4()}", task_queue=worker.task_queue, ) - assert message == ( - "Workflow code can't call a Stream in this version. Call it from an Activity " - "through activity.client()." + producer = await handle.result() + run_id = handle.result_run_id + assert run_id + assert producer == Producer( + id=f"wf:{run_id}", kind=ProducerKindWorkflow(WorkflowProducer(run_id=run_id)) ) + # On a Store, the append writes as the Workflow, with its Workflow Task as the epoch. + # The close passes the Store's check of that epoch too. + stream = stream_client.get_stream(StreamRef(id=stream_id), body_type=str) + [record] = await _records(stream) + assert record.producer is not None + assert record.producer.id == producer.id + assert isinstance(record.producer.kind, ProducerKindWorkflow) + assert record.producer.kind.value.epoch is not None + assert record.producer.kind.value.epoch.attempt == 1 + described = await stream.describe() + assert described.status == StreamStatus.StreamStatusClosed @pytest.mark.parametrize("kind", ["workflow", *_ACTIVITY_INFOS, "client"]) diff --git a/tests/helpers/streams.py b/tests/helpers/streams.py index f2a3aad34e..ba0593bed6 100644 --- a/tests/helpers/streams.py +++ b/tests/helpers/streams.py @@ -3,7 +3,6 @@ from __future__ import annotations import asyncio -import contextlib import dataclasses from collections.abc import Awaitable, Callable from typing import TypeVar @@ -34,7 +33,7 @@ ReadRecordsRequest, ReadRecordsResponse, ) -from temporalio.client import StreamStore +from temporalio.client import StreamBatchLimits, StreamStore from temporalio.exceptions import StreamError _RequestT = TypeVar("_RequestT", bound=Message) @@ -106,6 +105,12 @@ class _DictStream: closed: bool = False # The id of a Producer is unique in one Stream. last_batches: dict[str, _LastBatch] = dataclasses.field(default_factory=dict) + changed: asyncio.Event = dataclasses.field(default_factory=asyncio.Event) + + def wake(self) -> None: + """Wake the reads that wait, and give later reads a new event.""" + self.changed.set() + self.changed = asyncio.Event() def _offset(index: int) -> bytes: @@ -142,19 +147,27 @@ class DictStreamStore(StreamStore): It has no Retention and no expiry, and it checks no request. It resends the answer of a batch that comes again with the same Producer, epoch and sequence, and it - fences a Producer whose epoch is lower than its newest one. A read waits for a new - Record up to the ``wait`` of its request. + fences a Producer whose epoch is lower than its newest one. A read with a ``wait`` + waits for an append or a close, as Core needs for Workflow code. + :py:attr:`waiting` counts the reads that wait, so a test sees one that stays. """ - def __init__(self, name: str = "dict") -> None: + def __init__( + self, name: str = "dict", *, limits: StreamBatchLimits | None = None + ) -> None: self._name = name + self._limits = limits if limits is not None else StreamBatchLimits() self.streams: dict[str, _DictStream] = {} - self._changed = asyncio.Condition() + self.waiting = 0 @property def name(self) -> str: return self._name + @property + def limits(self) -> StreamBatchLimits: + return self._limits + def _stream(self, ref: StreamRef) -> _DictStream: stream = self.streams.get(ref.id) if stream is None: @@ -164,10 +177,6 @@ def _stream(self, ref: StreamRef) -> _DictStream: ) return stream - async def _notify(self) -> None: - async with self._changed: - self._changed.notify_all() - async def create_stream(self, request: CreateStreamRequest) -> CreateStreamResponse: ref = StreamRef(id=request.stream_ref.id, store=self.name) stream = self.streams.get(ref.id) @@ -219,22 +228,22 @@ async def append_records( first_offset=_offset(first), last_offset=_offset(len(stream.records)) ) stream.last_batches[request.producer.id] = _LastBatch(epoch, sequence, response) - await self._notify() + if request.records: + stream.wake() return response async def read_records(self, request: ReadRecordsRequest) -> ReadRecordsResponse: stream = self._stream(request.ref) start = int.from_bytes(request.from_offset, "big") if request.from_offset else 0 - wait = request.wait.ToTimedelta() - if wait and start >= len(stream.records) and not stream.closed: - async with self._changed: - with contextlib.suppress(TimeoutError): - await asyncio.wait_for( - self._changed.wait_for( - lambda: len(stream.records) > start or stream.closed - ), - wait.total_seconds(), - ) + wait = request.wait.ToTimedelta().total_seconds() + if start >= len(stream.records) and not stream.closed and wait > 0: + self.waiting += 1 + try: + await asyncio.wait_for(stream.changed.wait(), wait) + except TimeoutError: + pass + finally: + self.waiting -= 1 page = stream.records[start : start + (request.max_records or 1000)] last_offset = page[-1].offset if page else request.from_offset return ReadRecordsResponse( @@ -244,8 +253,9 @@ async def read_records(self, request: ReadRecordsRequest) -> ReadRecordsResponse ) async def close_stream(self, request: CloseStreamRequest) -> CloseStreamResponse: - self._stream(request.ref).closed = True - await self._notify() + stream = self._stream(request.ref) + stream.closed = True + stream.wake() return CloseStreamResponse() async def describe_stream( diff --git a/tests/nexus/test_stream_direct.py b/tests/nexus/test_stream_direct.py new file mode 100644 index 0000000000..34497767cb --- /dev/null +++ b/tests/nexus/test_stream_direct.py @@ -0,0 +1,2842 @@ +"""Reads and appends of a Stream that Core makes inside the Workflow Task.""" + +from __future__ import annotations + +import asyncio +import uuid +from collections.abc import AsyncGenerator, Sequence +from dataclasses import dataclass +from datetime import timedelta +from typing import Any + +import pytest + +from temporalio import activity, workflow +from temporalio.api.common.v1 import Payload +from temporalio.api.enums.v1 import EventType, StreamErrorReason +from temporalio.api.streamservice.v1 import ( + AppendRecordsRequest, + AppendRecordsResponse, + ReadRecordsRequest, + ReadRecordsResponse, +) +from temporalio.bridge.proto.external_data import ( + StreamCloseEntry, + StreamMarkerData, + StreamReadEntry, +) +from temporalio.client import ( + Client, + StreamBatchLimits, + StreamConfig, + StreamReplaySource, + WorkflowFailureError, + WorkflowHandle, + WorkflowHistory, +) +from temporalio.common import RawValue +from temporalio.converter import ( + DataConverter, + PayloadCodec, + SerializationContext, + WithSerializationContext, + WorkflowSerializationContext, +) +from temporalio.exceptions import ApplicationError, StreamError, StreamReplayError +from temporalio.nexus.system import stream_service +from temporalio.nexus.system.stream_service import ( + Producer, + ProducerKindWorkflow, + StreamRecord, + StreamRef, + StreamStatus, +) +from temporalio.runtime import MetricBuffer, Runtime, TelemetryConfig +from temporalio.testing import WorkflowEnvironment +from temporalio.worker import ( + Interceptor, + Replayer, + WorkflowInboundInterceptor, + WorkflowInterceptorClassInput, + WorkflowOutboundInterceptor, +) +from tests.helpers import assert_eventually, assert_task_fail_eventually, new_worker +from tests.helpers.streams import DictStreamStore, RecordingStreamStore +from tests.test_converter import SimpleCodec + +# A Record body that a test can look for in the bytes of History. +_MARK = "direct-read-mark" + + +def _stream_client( + client: Client, + store: Any = None, + *, + replay_from: StreamReplaySource = StreamReplaySource.STORE, + codec: bool = False, +) -> Client: + config = client.config() + config["stream_config"] = StreamConfig( + stores=[store if store is not None else DictStreamStore()], + workflow_read_replay_from=replay_from, + ) + if codec: + config["data_converter"] = DataConverter(payload_codec=SimpleCodec()) + return Client(**config) + + +def _markers(history: WorkflowHistory) -> list[str]: + return [ + event.marker_recorded_event_attributes.marker_name + for event in history.events + if event.event_type == EventType.EVENT_TYPE_MARKER_RECORDED + ] + + +def _stream_markers_per_task(history: WorkflowHistory) -> list[int]: + """Give the number of `core_stream` markers that each completed Workflow Task wrote.""" + counts: list[int] = [] + for event in history.events: + if event.event_type == EventType.EVENT_TYPE_WORKFLOW_TASK_COMPLETED: + counts.append(0) + elif ( + event.event_type == EventType.EVENT_TYPE_MARKER_RECORDED + and event.marker_recorded_event_attributes.marker_name == "core_stream" + ): + counts[-1] += 1 + return counts + + +def _completed_tasks(history: WorkflowHistory) -> int: + return sum( + 1 + for event in history.events + if event.event_type == EventType.EVENT_TYPE_WORKFLOW_TASK_COMPLETED + ) + + +async def _filled_stream( + client: Client, bodies: list[str], *, close: bool = True +) -> StreamRef[str]: + stream = await client.create_stream(id=f"stream-{uuid.uuid4()}", body_type=str) + if bodies: + await stream.append(*bodies) + if close: + await stream.close() + return stream.ref + + +async def _read_all(client: Client, ref: StreamRef[str]) -> list[StreamRecord[str]]: + stream = client.get_stream(ref, body_type=str) + return [record async for record in stream.read()] + + +async def _run( + client: Client, + workflow_type: type[Any], + *args: Any, + activities: list[Any] | None = None, + max_cached_workflows: int = 1000, + task_timeout: timedelta | None = None, +) -> tuple[WorkflowHandle[Any, Any], Any]: + async with new_worker( + client, + workflow_type, + activities=activities or [], + max_cached_workflows=max_cached_workflows, + ) as worker: + handle: WorkflowHandle[Any, Any] = await client.start_workflow( + workflow_type.run, + args=list(args), + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + task_timeout=task_timeout, + ) + return handle, await handle.result() + + +# A generator over `read` that reads the Stream of an Activity that still runs. + + +@dataclass +class ModelCall: + chunks_id: str + seen_id: str + + +@activity.defn +async def stream_model_output(call: ModelCall) -> None: + client = activity.client() + chunks = client.get_stream(StreamRef(id=call.chunks_id), body_type=str) + seen = client.get_stream(StreamRef(id=call.seen_id), body_type=str) + await chunks.append("Hel", "lo") + # The Activity writes the rest only after the Workflow says that it has the first + # chunks. So the test passes only if Workflow code gets chunks while the Activity runs. + told = False + async for _ in seen.read(wait=timedelta(seconds=30)): + told = True + break + if not told: + raise ApplicationError( + "The Workflow got no chunk while the Activity ran", non_retryable=True + ) + await chunks.append(", ", "world") + await chunks.close() + + +async def model_chunks(chunks_id: str) -> AsyncGenerator[str, None]: + """Yield each chunk of the model's Stream, as the model stub of a library does.""" + stream = workflow.get_stream(StreamRef(id=chunks_id), body_type=str) + # The wait is far longer than a pause of the model, so no page comes back empty. + async for record in stream.read(wait=timedelta(seconds=30)): + yield record.body + + +@workflow.defn +class LiveModelWorkflow: + @workflow.run + async def run(self, call: ModelCall) -> str: + model = workflow.start_activity( + stream_model_output, call, start_to_close_timeout=timedelta(seconds=60) + ) + seen = workflow.get_stream(StreamRef(id=call.seen_id), body_type=str) + text = "" + async for chunk in model_chunks(call.chunks_id): + text += chunk + if text == "Hello": + await seen.append("the first chunks") + await model + return text + + +async def test_generator_over_read_reads_a_running_activity( + client: Client, +): + stream_client = _stream_client(client) + chunks = await _filled_stream(stream_client, [], close=False) + seen = await _filled_stream(stream_client, [], close=False) + handle, text = await _run( + stream_client, + LiveModelWorkflow, + ModelCall(chunks.id, seen.id), + activities=[stream_model_output], + ) + assert text == "Hello, world" + history = await handle.fetch_history() + # The reads and the append write no Local Activity marker, and at most one marker for + # each Workflow Task. + assert "core_local_activity" not in _markers(history) + assert "core_stream" in _markers(history) + assert max(_stream_markers_per_task(history)) == 1 + + +# The pages of a read. + + +@dataclass +class ReadArgs: + stream_id: str + from_offset: str | None = None + wait_ms: int | None = None + max_records: int | None = None + max_bytes: int | None = None + store: str = "dict" + pages: int | None = None + """The most pages to read. ``None`` reads until the iterator ends.""" + + +@dataclass +class ReadResult: + bodies: list[str] + offsets: list[str] + page_sizes: list[int] + last_offsets: list[str | None] + dones: list[bool] + + +def _ms(value: int | None) -> timedelta | None: + return None if value is None else timedelta(milliseconds=value) + + +def _hex(value: bytes | None) -> str | None: + return None if value is None else value.hex() + + +@workflow.defn +class ReadPagesWorkflow: + @workflow.run + async def run(self, args: ReadArgs) -> ReadResult: + # The ref has its Store, so a replay with no Stream config knows it. + ref = StreamRef[str](id=args.stream_id, store=args.store) + stream = workflow.get_stream(ref, body_type=str) + result = ReadResult([], [], [], [], []) + async for page in stream.read_pages( + from_offset=( + bytes.fromhex(args.from_offset) + if args.from_offset is not None + else None + ), + max_records=args.max_records, + wait=_ms(args.wait_ms), + max_bytes=args.max_bytes, + ): + result.bodies.extend(record.body for record in page.records) + result.offsets.extend(record.offset.hex() for record in page.records) + result.page_sizes.append(len(page.records)) + result.last_offsets.append(_hex(page.last_offset)) + result.dones.append(page.done) + if len(result.page_sizes) == args.pages: + break + return result + + +async def _read_pages( + client: Client, args: ReadArgs, *, max_cached_workflows: int = 1000 +) -> ReadResult: + _, result = await _run( + client, ReadPagesWorkflow, args, max_cached_workflows=max_cached_workflows + ) + return result + + +@pytest.mark.parametrize("max_cached_workflows", [1000, 0]) +async def test_closed_stream_ends_the_iterator_with_a_done_page( + client: Client, max_cached_workflows: int +): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a", "b", "c"]) + result = await _read_pages( + stream_client, + ReadArgs(ref.id, wait_ms=30_000), + max_cached_workflows=max_cached_workflows, + ) + assert result.bodies == ["a", "b", "c"] + # The iterator ends after the page with `done`, also with a `wait`. + assert result.dones[-1] is True + assert not any(result.dones[:-1]) + assert result.last_offsets[-1] == result.offsets[-1] + + +@pytest.mark.parametrize("max_cached_workflows", [1000, 0]) +async def test_read_with_max_records_gives_pages_of_two_two_and_one( + client: Client, max_cached_workflows: int +): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a", "b", "c", "d", "e"]) + result = await _read_pages( + stream_client, + ReadArgs(ref.id, max_records=2), + max_cached_workflows=max_cached_workflows, + ) + assert result.bodies == ["a", "b", "c", "d", "e"] + assert result.page_sizes == [2, 2, 1] + assert result.dones == [False, False, True] + # Each page ends at its last Record, and the next one starts after it. + assert result.last_offsets == [ + result.offsets[1], + result.offsets[3], + result.offsets[4], + ] + + +@workflow.defn +class ReadOnePageWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str], wait_ms: int | None) -> list[Any]: + stream = workflow.get_stream(ref, body_type=str) + page = await stream.read_page(wait=_ms(wait_ms)) + return [[record.body for record in page.records], page.done, page.last_offset] + + +@pytest.mark.parametrize("max_cached_workflows", [1000, 0]) +async def test_read_with_wait_on_a_quiet_stream_gives_an_empty_page( + client: Client, max_cached_workflows: int +): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, [], close=False) + handle, result = await _run( + stream_client, + ReadOnePageWorkflow, + ref, + 1000, + max_cached_workflows=max_cached_workflows, + ) + assert result == [[], False, None] + # The page comes only after the wait. + events = (await handle.fetch_history()).events + elapsed = events[-1].event_time.ToDatetime() - events[0].event_time.ToDatetime() + assert elapsed >= timedelta(seconds=1) + + +async def test_read_with_no_wait_on_an_empty_stream_gives_an_empty_page( + client: Client, +): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, [], close=False) + handle, result = await _run(stream_client, ReadOnePageWorkflow, ref, None) + assert result == [[], False, None] + events = (await handle.fetch_history()).events + # An empty Stream gives no reason to wait. + took = events[-1].event_time.ToDatetime() - events[0].event_time.ToDatetime() + assert took < timedelta(seconds=5) + + +@pytest.mark.parametrize("max_cached_workflows", [1000, 0]) +async def test_read_with_no_wait_gives_what_is_there_and_ends( + client: Client, max_cached_workflows: int +): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a", "b", "c"], close=False) + handle, bodies = await _run( + stream_client, + ReadItemsWorkflow, + ref, + None, + max_cached_workflows=max_cached_workflows, + ) + assert bodies == ["a", "b", "c"] + events = (await handle.fetch_history()).events + # The read doesn't wait for the Store, so the run ends long before the first heartbeat. + took = events[-1].event_time.ToDatetime() - events[0].event_time.ToDatetime() + assert took < timedelta(seconds=5) + + +@workflow.defn +class ReadItemsWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str], wait_ms: int | None) -> list[str]: + stream = workflow.get_stream(ref, body_type=str) + return [record.body async for record in stream.read(wait=_ms(wait_ms))] + + +async def test_page_that_takes_the_last_record_with_a_cap_is_done(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a", "b", "c"]) + # The cap takes the last Record of the closed Stream, so the page is done. + by_records = await _read_pages(stream_client, ReadArgs(ref.id, max_records=3)) + assert (by_records.page_sizes, by_records.dones) == ([3], [True]) + one = await _filled_stream(stream_client, ["a"]) + by_bytes = await _read_pages(stream_client, ReadArgs(one.id, max_bytes=1)) + assert (by_bytes.page_sizes, by_bytes.dones) == ([1], [True]) + # On an open Stream the same page isn't done. + open_ref = await _filled_stream(stream_client, ["a", "b", "c"], close=False) + capped = await _read_pages( + stream_client, ReadArgs(open_ref.id, max_records=3, pages=1) + ) + assert (capped.page_sizes, capped.dones) == ([3], [False]) + + +async def test_page_with_max_bytes_gives_the_first_record_over_the_cap( + client: Client, +): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a", "b", "c"]) + result = await _read_pages(stream_client, ReadArgs(ref.id, max_bytes=1)) + # The first Record of each page always comes, even over the cap. + assert result.page_sizes == [1, 1, 1] + assert result.dones == [False, False, True] + + +@activity.defn +async def append_after_a_quiet_gap(stream_id: str) -> None: + stream = activity.client().get_stream(StreamRef(id=stream_id), body_type=str) + await stream.append("a") + await asyncio.sleep(2) + await stream.append("b") + await stream.close() + + +@workflow.defn +class QuietGapWorkflow: + @workflow.run + async def run(self, stream_id: str) -> list[str]: + writer = workflow.start_activity( + append_after_a_quiet_gap, + stream_id, + start_to_close_timeout=timedelta(seconds=30), + ) + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + got = [record.body async for record in stream.read(wait=timedelta(seconds=10))] + await writer + return got + + +async def test_read_with_a_wait_reads_across_a_quiet_gap(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, [], close=False) + handle, result = await _run( + stream_client, + QuietGapWorkflow, + ref.id, + activities=[append_after_a_quiet_gap], + ) + assert result == ["a", "b"] + # The gap of 2 s is shorter than the wait, so one Core read gets both Records. + assert len(_read_seqs(await handle.fetch_history())) == 1 + + +# A loop with no await in its body, and one that awaits for each Record. + + +def _read_entries(history: WorkflowHistory) -> list[list[StreamReadEntry]]: + """Give the read entries of the `core_stream` marker of each Workflow Task.""" + entries: list[list[StreamReadEntry]] = [] + for event in history.events: + attributes = event.marker_recorded_event_attributes + if ( + event.event_type != EventType.EVENT_TYPE_MARKER_RECORDED + or attributes.marker_name != "core_stream" + ): + continue + [payload] = attributes.details["data"].payloads + entries.append(list(StreamMarkerData.FromString(payload.data).reads)) + return entries + + +def _read_seqs(history: WorkflowHistory) -> set[int]: + """Give the seq of each Core read that History has.""" + return {entry.seq for task in _read_entries(history) for entry in task} + + +@activity.defn +async def append_five_slowly(stream_id: str) -> None: + stream = activity.client().get_stream(StreamRef(id=stream_id), body_type=str) + for index in range(5): + await asyncio.sleep(0.3) + await stream.append(f"record-{index}") + await stream.close() + + +@workflow.defn +class LoopWithNoAwaitWorkflow: + @workflow.run + async def run(self, stream_id: str) -> list[str]: + writer = workflow.start_activity( + append_five_slowly, stream_id, start_to_close_timeout=timedelta(seconds=30) + ) + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + got = [] + async for record in stream.read(wait=timedelta(seconds=10)): + got.append(record.body) + await writer + return got + + +@pytest.mark.parametrize("max_cached_workflows", [1000, 0]) +async def test_loop_with_no_await_keeps_one_core_read( + client: Client, max_cached_workflows: int +): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, [], close=False) + handle, got = await _run( + stream_client, + LoopWithNoAwaitWorkflow, + ref.id, + activities=[append_five_slowly], + max_cached_workflows=max_cached_workflows, + ) + assert got == [f"record-{index}" for index in range(5)] + entries = _read_entries(await handle.fetch_history()) + # The loop asks for the next page in the activation of the last one, so one Core + # read gives every page. + assert len({entry.seq for task in entries for entry in task}) == 1 + assert any(len(task) == 1 and len(task[0].page_counts) > 1 for task in entries) + + +@workflow.defn +class LoopWithAppendWorkflow: + @workflow.run + async def run(self, stream_id: str, out_id: str) -> list[str]: + writer = workflow.start_activity( + append_five_slowly, stream_id, start_to_close_timeout=timedelta(seconds=30) + ) + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + out = workflow.get_stream(StreamRef(id=out_id), body_type=str) + got = [] + async for record in stream.read(wait=timedelta(seconds=10)): + got.append(record.body) + await out.append(record.body) + await writer + return got + + +@pytest.mark.parametrize("max_cached_workflows", [1000, 0]) +async def test_loop_that_awaits_an_append_closes_and_starts_the_read( + client: Client, max_cached_workflows: int +): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, [], close=False) + out_ref = await _filled_stream(stream_client, [], close=False) + handle, got = await _run( + stream_client, + LoopWithAppendWorkflow, + ref.id, + out_ref.id, + activities=[append_five_slowly], + max_cached_workflows=max_cached_workflows, + ) + bodies = [f"record-{index}" for index in range(5)] + assert got == bodies + # Each Record went to the other Stream once and in order, whatever the replays. + assert [record.body for record in await _read_all(stream_client, out_ref)] == bodies + # The append ends the activation with no call waiting on the read, so the read + # closes, and the next page starts another one. + assert len(_read_seqs(await handle.fetch_history())) > 1 + + +# The example of the design review: +# +# let mut i = 0; +# while let item = stream.next() { +# workflow.spawn(loop { yield_once.await; i++; }) +# if i == 2 { await execute_activity(...); } +# } +# +# Each Record starts one more task that counts the wakes of the Workflow. So whether the +# Activity starts, and at which Record, depends on the activations that gave the Workflow +# its Records, and not on the Records. + + +class _Wakes: + """Lets a task of a Workflow wait for the next wake of the Workflow. + + Python has no call for it. ``asyncio.sleep(0)`` resumes in the same activation, so a + loop of it never lets the activation end. + """ + + def __init__(self) -> None: + self._new: list[asyncio.Future[None]] = [] + self._parked: list[asyncio.Future[None]] = [] + self._resumed = False + self._watch = asyncio.create_task(workflow.wait_condition(self._on_idle)) + + def _on_idle(self) -> bool: + # The event loop checks a condition each time it has no ready task. That is once at + # the end of a wake, and once more when the tasks that this resumed have run. The + # second check is in the same wake, so it resumes nothing. + if self._resumed: + self._resumed = False + self._parked += self._new + else: + self._resumed = bool(self._parked) + for future in self._parked: + if not future.done(): + future.set_result(None) + self._parked = self._new + self._new = [] + return False + + async def yield_once(self) -> None: + future: asyncio.Future[None] = asyncio.get_running_loop().create_future() + self._new.append(future) + await future + + +@workflow.defn +class CounterTasksWorkflow: + def __init__(self) -> None: + self.i = 0 + + async def count(self, wakes: _Wakes) -> None: + while True: + await wakes.yield_once() + self.i += 1 + + @workflow.run + async def run(self, stream_id: str) -> list[int]: + wakes = _Wakes() + seen: list[int] = [] + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + async for item in stream.read(wait=timedelta(seconds=30)): + asyncio.create_task(self.count(wakes)) + seen.append(self.i) + if self.i == 2: + await workflow.execute_activity( + tag, item.body, start_to_close_timeout=timedelta(seconds=10) + ) + return seen + + +async def _read_calls(count: int) -> None: + """Wait until Workflow code has made this many `read_page` calls.""" + + async def check() -> None: + assert [name for name, _ in _intercepted].count("ReadRecords") >= count + + await assert_eventually(check, interval=timedelta(milliseconds=20)) + + +async def _run_counter_tasks( + client: Client, pages: list[list[str]], *, max_cached_workflows: int = 1000 +) -> tuple[list[int], WorkflowHistory]: + """Run the example, and give it each page of Records in an activation of its own. + + The last page must make the Workflow start its Activity. + """ + _intercepted.clear() + stream = await client.create_stream(id=f"stream-{uuid.uuid4()}", body_type=str) + # The Records that the Stream has when the read starts come in one page. + await stream.append(*pages[0]) + async with new_worker( + client, + CounterTasksWorkflow, + activities=[tag], + interceptors=[StreamCallInterceptor()], + max_cached_workflows=max_cached_workflows, + ) as worker: + handle = await client.start_workflow( + CounterTasksWorkflow.run, + stream.ref.id, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + for calls, page in enumerate(pages[1:], start=2): + # The loop asks for its next page in the activation that gave it the page + # before. A Record that comes after that call is in a later activation. + await _read_calls(calls) + await stream.append(*page) + + async def activity_started() -> None: + assert _scheduled_activities(await handle.fetch_history()) == ["tag"] + + await assert_eventually(activity_started) + await stream.close() + seen = await handle.result() + return seen, await handle.fetch_history() + + +_COUNTER_PAGES = [["record-0", "record-1"], ["record-2"], ["record-3"]] + + +@pytest.mark.parametrize("max_cached_workflows", [1000, 0]) +async def test_counter_tasks_start_the_activity_by_the_activations( + client: Client, max_cached_workflows: int +): + stream_client = _stream_client(client) + seen, history = await _run_counter_tasks( + stream_client, _COUNTER_PAGES, max_cached_workflows=max_cached_workflows + ) + # The first two Records start a task each, and both tasks count the next wake. The loop + # reads `i` before the tasks of a wake run, so `i` is 2 at the fourth Record. + assert seen == [0, 0, 0, 2] + # The marker of the Workflow Task that started the Activity has the three pages, and + # each page is in an activation of its own. + [entry] = _read_entries(history)[0] + assert list(entry.page_counts) == [2, 1, 1] + assert all(gap > 0 for gap in entry.page_activations[1:]) + # A replay gives each page in the activation that got it, so `i` is the same at each + # Record. With no cache, the Workflow Tasks after the first one were such replays too. + config = stream_client.config()["stream_config"] + await Replayer( + workflows=[CounterTasksWorkflow], stream_config=config + ).replay_workflow(history) + + +async def test_counter_tasks_with_one_page_start_no_activity(client: Client): + stream_client = _stream_client(client) + bodies = [body for page in _COUNTER_PAGES for body in page] + ref = await _filled_stream(stream_client, bodies) + handle, seen = await _run( + stream_client, CounterTasksWorkflow, ref.id, activities=[tag] + ) + # With the same Records in one activation, no task counts a wake between two of them. + assert seen == [0, 0, 0, 0] + assert _scheduled_activities(await handle.fetch_history()) == [] + + +async def test_replay_with_other_activations_is_a_nondeterminism_error(client: Client): + stream_client = _stream_client(client) + seen, history = await _run_counter_tasks(stream_client, _COUNTER_PAGES) + assert seen == [0, 0, 0, 2] + # Put every page of a read into the activation of its first page. `i` then stays 0, and + # the replay does not start the Activity that History has. + for event in history.events: + attributes = event.marker_recorded_event_attributes + if ( + event.event_type != EventType.EVENT_TYPE_MARKER_RECORDED + or attributes.marker_name != "core_stream" + ): + continue + [payload] = attributes.details["data"].payloads + data = StreamMarkerData.FromString(payload.data) + for entry in data.reads: + gaps = [0] * (len(entry.page_activations) - 1) + del entry.page_activations[1:] + entry.page_activations.extend(gaps) + payload.data = data.SerializeToString() + config = stream_client.config()["stream_config"] + replay = await Replayer( + workflows=[CounterTasksWorkflow], stream_config=config + ).replay_workflow(history, raise_on_replay_failure=False) + assert isinstance(replay.replay_failure, workflow.NondeterminismError) + + +# The Records of one page, taken in several activations. Core records the page +# when it gives it. An Activity between two Records is a later Workflow Task, +# and that task has no marker. + + +@workflow.defn +class ActivityForEachRecordWorkflow: + @workflow.run + async def run(self, stream_id: str) -> list[str]: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + got = [] + async for item in stream.read(): + got.append( + await workflow.execute_activity( + tag, item.body, start_to_close_timeout=timedelta(seconds=10) + ) + ) + return got + + +@workflow.defn +class OneRecordPerReadWorkflow: + @workflow.run + async def run(self, stream_id: str) -> list[str]: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + got = [] + offset = None + while True: + page = await stream.read_page(from_offset=offset, max_records=1) + for record in page.records: + got.append( + await workflow.execute_activity( + tag, + record.body, + start_to_close_timeout=timedelta(seconds=10), + ) + ) + offset = page.last_offset + if page.done or not page.records: + return got + + +async def _one_page_across_activations( + client: Client, workflow_type: type[Any], *, max_cached_workflows: int +) -> None: + ref = await _filled_stream(client, ["a", "b", "c", "d"]) + handle, got = await _run( + client, + workflow_type, + ref.id, + activities=[tag], + max_cached_workflows=max_cached_workflows, + ) + assert got == [f"tagged {body}" for body in "abcd"] + history = await handle.fetch_history() + # The last Activity's result is a task of its own, and that task completes + # the Workflow. It records nothing, so the marker stays in the first task. + assert _completed_tasks(history) == 5 + assert _scheduled_activities(history) == ["tag", "tag", "tag", "tag"] + task = 0 + marker_task = None + for event in history.events: + if event.event_type == EventType.EVENT_TYPE_WORKFLOW_TASK_COMPLETED: + task += 1 + elif ( + event.event_type == EventType.EVENT_TYPE_MARKER_RECORDED + and event.marker_recorded_event_attributes.marker_name == "core_stream" + ): + assert marker_task is None + marker_task = task + assert marker_task == 1 + entries = _read_entries(history) + assert len(entries) == 1 + [entry] = entries[0] + assert list(entry.page_counts) == [4] + assert list(entry.page_activations) == [1] + config = client.config()["stream_config"] + await Replayer(workflows=[workflow_type], stream_config=config).replay_workflow( + history + ) + + +@pytest.mark.parametrize("max_cached_workflows", [1000, 0]) +async def test_one_page_across_activities_keeps_its_marker( + client: Client, max_cached_workflows: int +): + await _one_page_across_activations( + _stream_client(client), + ActivityForEachRecordWorkflow, + max_cached_workflows=max_cached_workflows, + ) + + +@pytest.mark.parametrize("max_cached_workflows", [1000, 0]) +async def test_one_record_reads_of_one_page_keep_its_marker( + client: Client, max_cached_workflows: int +): + await _one_page_across_activations( + _stream_client(client), + OneRecordPerReadWorkflow, + max_cached_workflows=max_cached_workflows, + ) + + +# What `read_page` checks, and where it can't run. + + +@workflow.defn +class LongOffsetWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str]) -> list[str]: + stream = workflow.get_stream(ref, body_type=str) + got: list[str] = [] + try: + await stream.read_page(from_offset=b"o" * 2000) + got.append("page") + except StreamError as err: + got.append(StreamErrorReason.Name(err.reason)) + got.extend([record.body async for record in stream.read()]) + return got + + +async def test_read_from_an_offset_over_1024_bytes_raises_invalid(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a", "b"]) + handle, got = await _run(stream_client, LongOffsetWorkflow, ref) + assert got == ["STREAM_ERROR_REASON_INVALID", "a", "b"] + config = stream_client.config()["stream_config"] + await Replayer( + workflows=[LongOffsetWorkflow], stream_config=config + ).replay_workflow(history=await handle.fetch_history()) + + +@workflow.defn +class QueryReadWorkflow: + def __init__(self) -> None: + self.ref: StreamRef[str] | None = None + + @workflow.run + async def run(self, ref: StreamRef[str]) -> None: + self.ref = ref + await workflow.wait_condition(lambda: False) + + @workflow.query + def start_read(self) -> str: + assert self.ref is not None + step = workflow.get_stream(self.ref, body_type=str).read_page() + try: + step.send(None) + except Exception as err: + return type(err).__name__ + finally: + step.close() + return "started" + + +async def test_read_in_a_query_raises_the_read_only_error(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a"]) + async with new_worker(stream_client, QueryReadWorkflow) as worker: + handle = await stream_client.start_workflow( + QueryReadWorkflow.run, + ref, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + assert await handle.query(QueryReadWorkflow.start_read) == ( + "ReadOnlyContextError" + ) + await handle.terminate() + + +# Reads that resume from an offset. + + +@workflow.defn +class LoopReadWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str]) -> list[str]: + stream = workflow.get_stream(ref, body_type=str) + got: list[str] = [] + # The first loop leaves with `break`, and its next request resumes there. + items = stream.read(max_records=4) + async for record in items: + got.append(record.body) + if len(got) == 3: + break + assert items.next_request is not None + last = items.next_request.from_offset + # The last reads go on from there to the end, two Records at a time. + while True: + page = await stream.read_page(from_offset=last, max_records=2) + got.extend(record.body for record in page.records) + last = page.last_offset + if page.done: + return got + + +async def test_reads_from_an_offset_get_each_record_once(client: Client): + stream_client = _stream_client(client) + bodies = [f"record-{index}" for index in range(10)] + ref = await _filled_stream(stream_client, bodies) + _, got = await _run(stream_client, LoopReadWorkflow, ref) + assert got == bodies + + +# The marker in History. + + +async def _history_of_a_read( + client: Client, replay_from: StreamReplaySource +) -> tuple[WorkflowHistory, StreamRef[str], Client]: + stream_client = _stream_client(client, replay_from=replay_from) + ref = await _filled_stream(stream_client, [f"{_MARK}-1", f"{_MARK}-2"]) + handle, result = await _run(stream_client, ReadPagesWorkflow, ReadArgs(ref.id)) + assert result.bodies == [f"{_MARK}-1", f"{_MARK}-2"] + return await handle.fetch_history(), ref, stream_client + + +def _markers_have_mark(history: WorkflowHistory) -> bool: + # The result of the Workflow holds the bodies, so only the markers count. + return any( + _MARK.encode() in event.SerializeToString() + for event in history.events + if event.event_type == EventType.EVENT_TYPE_MARKER_RECORDED + ) + + +async def test_store_marker_holds_no_body(client: Client): + history, _, _ = await _history_of_a_read(client, StreamReplaySource.STORE) + assert "core_stream" in _markers(history) + assert not _markers_have_mark(history) + + +async def test_history_marker_holds_the_bodies(client: Client): + history, _, _ = await _history_of_a_read(client, StreamReplaySource.HISTORY) + assert "core_stream" in _markers(history) + assert _markers_have_mark(history) + + +# Replay. + + +async def test_replay_of_store_marker_reads_the_store(client: Client): + history, _, stream_client = await _history_of_a_read( + client, StreamReplaySource.STORE + ) + config = stream_client.config()["stream_config"] + await Replayer(workflows=[ReadPagesWorkflow], stream_config=config).replay_workflow( + history + ) + + +async def _failed_replay( + history: WorkflowHistory, ref: StreamRef[str], config: StreamConfig | None +) -> str: + """Replay with and without a raise, and give the text of the failure.""" + replayer = Replayer(workflows=[ReadPagesWorkflow], stream_config=config) + with pytest.raises(RuntimeError) as err: + await replayer.replay_workflow(history) + result = await replayer.replay_workflow(history, raise_on_replay_failure=False) + assert isinstance(result.replay_failure, RuntimeError) + assert str(result.replay_failure) == str(err.value) + message = str(err.value) + assert f"StreamReplayError: Stream `{ref.id}` in Store `dict`" in message + return message + + +async def test_replay_of_store_marker_with_no_stream_config_fails(client: Client): + history, ref, _ = await _history_of_a_read(client, StreamReplaySource.STORE) + message = await _failed_replay(history, ref, None) + assert "this Worker has no Stream runtime for the Store `dict`" in message + + +async def test_replay_of_store_marker_fails_when_records_are_gone(client: Client): + history, ref, _ = await _history_of_a_read(client, StreamReplaySource.STORE) + # A new Store of the same name has none of the Records. + message = await _failed_replay( + history, ref, StreamConfig(stores=[DictStreamStore()]) + ) + assert "stream_not_found" in message + + +async def test_replay_of_history_marker_needs_no_stream_config(client: Client): + history, _, _ = await _history_of_a_read(client, StreamReplaySource.HISTORY) + await Replayer(workflows=[ReadPagesWorkflow]).replay_workflow(history) + + +# Two reads at once, on a Worker with no cache and on a cached one. + + +@activity.defn +async def tag(body: str) -> str: + return f"tagged {body}" + + +@dataclass +class TwoReadsArgs: + first_id: str + second_id: str + out_id: str + + +@dataclass +class TwoReadsResult: + first: list[str] + second: list[str] + count: int + reads: int + + +@workflow.defn +class TwoReadsWorkflow: + def __init__(self) -> None: + self.count = 0 + self.reads = 0 + + async def _read(self, stream_id: str, out: Any) -> list[str]: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + got: list[str] = [] + last = None + while True: + self.reads += 1 + page = await stream.read_page(from_offset=last, max_records=2) + for record in page.records: + tagged = await workflow.execute_local_activity( + tag, record.body, start_to_close_timeout=timedelta(seconds=10) + ) + await out.append(tagged) + self.count += 1 + got.append(tagged) + last = page.last_offset + if page.done: + return got + # A timer ends the Workflow Task, so a Worker with no cache replays each read. + await workflow.sleep(0.05) + + @workflow.run + async def run(self, args: TwoReadsArgs) -> TwoReadsResult: + out = workflow.get_stream(StreamRef(id=args.out_id), body_type=str) + first, second = await asyncio.gather( + self._read(args.first_id, out), self._read(args.second_id, out) + ) + return TwoReadsResult(first, second, self.count, self.reads) + + +@pytest.mark.parametrize("max_cached_workflows", [1000, 0]) +async def test_two_reads_with_and_without_a_cache( + client: Client, max_cached_workflows: int +): + stream_client = _stream_client(client) + first = [f"first-{index}" for index in range(5)] + second = [f"second-{index}" for index in range(5)] + out_ref = await _filled_stream(stream_client, [], close=False) + args = TwoReadsArgs( + first_id=(await _filled_stream(stream_client, first)).id, + second_id=(await _filled_stream(stream_client, second)).id, + out_id=out_ref.id, + ) + handle, result = await _run( + stream_client, + TwoReadsWorkflow, + args, + activities=[tag], + max_cached_workflows=max_cached_workflows, + ) + assert result == TwoReadsResult( + first=[f"tagged {body}" for body in first], + second=[f"tagged {body}" for body in second], + count=10, + reads=6, + ) + # Each Record went to the out Stream once, whatever the replays. + out = [record.body for record in await _read_all(stream_client, out_ref)] + assert sorted(out) == sorted(result.first + result.second) + assert _completed_tasks(await handle.fetch_history()) > 2 + + +# A Direct append. + + +@workflow.defn +class AppendWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str]) -> list[str | None]: + stream = workflow.get_stream(ref, body_type=str) + appended = await stream.append("a", "b") + # A timer ends the Workflow Task, so a Worker with no cache replays the append. + await workflow.sleep(0.05) + return [_hex(appended.first_offset), _hex(appended.last_offset)] + + +async def test_direct_append(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, [], close=False) + handle, offsets = await _run( + stream_client, AppendWorkflow, ref, max_cached_workflows=0 + ) + records = await _read_all(stream_client, ref) + # The replay of the next Workflow Task appended nothing. + assert [record.body for record in records] == ["a", "b"] + assert offsets == [record.offset.hex() for record in records] + history = await handle.fetch_history() + run_id = handle.result_run_id + started = [ + event.event_id + for event in history.events + if event.event_type == EventType.EVENT_TYPE_WORKFLOW_TASK_STARTED + ] + for record in records: + producer = record.producer + assert isinstance(producer, Producer) + assert producer.id == f"wf:{run_id}" + assert isinstance(producer.kind, ProducerKindWorkflow) + assert producer.kind.value.run_id == run_id + epoch = producer.kind.value.epoch + assert epoch is not None + # The epoch is the first Workflow Task, which made the append. + assert (epoch.started_event_id, epoch.attempt) == (started[0], 1) + assert _markers(history) == ["core_stream"] + # A Replayer appends nothing either. + await Replayer( + workflows=[AppendWorkflow], + stream_config=stream_client.config()["stream_config"], + ).replay_workflow(history) + assert len(await _read_all(stream_client, ref)) == 2 + + +@workflow.defn +class RefusedAppendWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str]) -> list[str]: + stream = workflow.get_stream(ref, body_type=str) + try: + await stream.append("a") + except StreamError as err: + # A timer only on the error path, so a replay without the error differs. + await workflow.sleep(0.01) + return [ + StreamErrorReason.Name(err.reason), + err.message, + str(err.non_retryable), + ] + return ["appended"] + + +async def test_refused_append_raises_stream_error_live_and_in_replay(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a"]) + handle, result = await _run(stream_client, RefusedAppendWorkflow, ref) + # The client path gives the same error for the same append. + with pytest.raises(StreamError) as client_err: + await stream_client.get_stream(ref, body_type=str).append("a") + assert result == [ + "STREAM_ERROR_REASON_CLOSED", + client_err.value.message, + "True", + ] + await Replayer( + workflows=[RefusedAppendWorkflow], + stream_config=stream_client.config()["stream_config"], + ).replay_workflow(await handle.fetch_history()) + + +# A Workflow Task that fails once in the middle of a read. + +_fail_read_task = True + + +@workflow.defn(sandboxed=False) +class FailOnceInReadWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str], out_ref: StreamRef[str]) -> list[str]: + stream = workflow.get_stream(ref, body_type=str) + out = workflow.get_stream(out_ref, body_type=str) + got: list[str] = [] + async for record in stream.read(): + got.append(record.body) + if record.body == "record-2": + await out.append("seen record-2") + global _fail_read_task + if _fail_read_task: + _fail_read_task = False + raise RuntimeError("Failed the Workflow Task once") + return got + + +async def test_failed_task_in_a_read_gets_each_record_once(client: Client): + global _fail_read_task + _fail_read_task = True + stream_client = _stream_client(client) + bodies = [f"record-{index}" for index in range(5)] + ref = await _filled_stream(stream_client, bodies) + out_ref = await _filled_stream(stream_client, [], close=False) + _, got = await _run(stream_client, FailOnceInReadWorkflow, ref, out_ref) + assert not _fail_read_task + assert got == bodies + # The append of the failed task reached the Store, and its retry appends again. + out = await _read_all(stream_client, out_ref) + assert [record.body for record in out] == ["seen record-2"] * 2 + epochs = [] + for record in out: + assert record.producer is not None + assert isinstance(record.producer.kind, ProducerKindWorkflow) + epoch = record.producer.kind.value.epoch + assert epoch is not None + epochs.append((epoch.started_event_id, epoch.attempt)) + assert epochs[0] < epochs[1] + + +# A Signal while a read waits. + + +@workflow.defn +class SignalStopsReadWorkflow: + def __init__(self) -> None: + self.signaled = False + + @workflow.run + async def run(self, ref: StreamRef[str]) -> list[Any]: + stream = workflow.get_stream(ref, body_type=str) + got: list[str] = [] + + async def read_all() -> None: + async for record in stream.read(wait=timedelta(seconds=60)): + got.append(record.body) + + reader = asyncio.create_task(read_all()) + await workflow.wait_condition(lambda: self.signaled) + # The cancel leaves the read with no call that waits on it, so it closes. + reader.cancel() + try: + await reader + except asyncio.CancelledError: + pass + return [got, self.signaled] + + @workflow.signal + def stop(self) -> None: + self.signaled = True + + +async def test_signal_reaches_a_waiting_read(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a"], close=False) + async with new_worker(stream_client, SignalStopsReadWorkflow) as worker: + handle = await stream_client.start_workflow( + SignalStopsReadWorkflow.run, + ref, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + # The heartbeat of an open task comes at 80 % of its timeout. + task_timeout=timedelta(seconds=2), + ) + await asyncio.sleep(0.5) + await handle.signal(SignalStopsReadWorkflow.stop) + result = await asyncio.wait_for(handle.result(), timeout=30) + assert result == [["a"], True] + await Replayer( + workflows=[SignalStopsReadWorkflow], + stream_config=stream_client.config()["stream_config"], + ).replay_workflow(await handle.fetch_history()) + + +# A second loop after a `break`, and the ends of a run with a read open. + + +@activity.defn +async def append_and_close(stream_id: str) -> None: + stream = activity.client().get_stream(StreamRef(id=stream_id), body_type=str) + await stream.append("x", "y") + await stream.close() + + +@workflow.defn +class BreakThenReadAgainWorkflow: + @workflow.run + async def run(self, stream_id: str) -> list[Any]: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + first = stream.read(wait=timedelta(seconds=30)) + got_first: list[str] = [] + async for record in first: + got_first.append(record.body) + if len(got_first) == 2: + break + assert first.next_request is not None + # The second loop resumes after the last Record that the first one gave. + second = stream.read_pages( + from_offset=first.next_request.from_offset, wait=timedelta(seconds=30) + ) + writer = workflow.start_activity( + append_and_close, stream_id, start_to_close_timeout=timedelta(seconds=30) + ) + got_second: list[str] = [] + done = False + async for page in second: + got_second.extend(record.body for record in page.records) + done = page.done + await writer + return [got_first, got_second, done] + + +async def test_second_loop_after_a_break_gets_every_record(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a", "b", "c"], close=False) + handle, result = await _run( + stream_client, BreakThenReadAgainWorkflow, ref.id, activities=[append_and_close] + ) + assert result == [["a", "b"], ["c", "x", "y"], True] + await Replayer( + workflows=[BreakThenReadAgainWorkflow], + stream_config=stream_client.config()["stream_config"], + ).replay_workflow(await handle.fetch_history()) + + +@workflow.defn +class CompleteWithOpenReadsWorkflow: + @workflow.run + async def run(self, stream_id: str) -> str: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + first = await stream.read_page(wait=timedelta(seconds=60)) + # A second read waits in its own task while the Workflow returns. + waiting = asyncio.create_task( + stream.read_page(from_offset=first.last_offset, wait=timedelta(seconds=60)) + ) + await asyncio.sleep(0) + assert not waiting.done() + return "completed" + + +async def test_workflow_completes_with_open_reads(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a"], close=False) + async with new_worker(stream_client, CompleteWithOpenReadsWorkflow) as worker: + handle = await stream_client.start_workflow( + CompleteWithOpenReadsWorkflow.run, + ref.id, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + assert await asyncio.wait_for(handle.result(), timeout=20) == "completed" + await Replayer( + workflows=[CompleteWithOpenReadsWorkflow], + stream_config=stream_client.config()["stream_config"], + ).replay_workflow(await handle.fetch_history()) + + +@workflow.defn +class CancelDuringReadWorkflow: + @workflow.run + async def run(self, stream_id: str) -> list[Any]: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + got: list[str] = [] + try: + async for record in stream.read(wait=timedelta(seconds=60)): + got.append(record.body) + except asyncio.CancelledError: + return [got, "cancelled"] + return [got, "ended"] + + +async def test_cancel_reaches_a_waiting_read(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a"], close=False) + async with new_worker(stream_client, CancelDuringReadWorkflow) as worker: + handle = await stream_client.start_workflow( + CancelDuringReadWorkflow.run, + ref.id, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + # The heartbeat of an open task comes at 80 % of its timeout. + task_timeout=timedelta(seconds=2), + ) + await asyncio.sleep(0.5) + await handle.cancel() + result = await asyncio.wait_for(handle.result(), timeout=30) + assert result == [["a"], "cancelled"] + + +@workflow.defn +class ContinueWithOpenReadWorkflow: + @workflow.run + async def run(self, stream_id: str, from_offset: str | None) -> list[str]: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + if from_offset is None: + # The Core read of this page is still open when the run continues. + first = await stream.read_page(max_records=1, wait=timedelta(seconds=60)) + assert first.last_offset is not None + workflow.continue_as_new(args=[stream_id, first.last_offset.hex()]) + items = stream.read(from_offset=bytes.fromhex(from_offset)) + return [record.body async for record in items] + + +async def test_continue_as_new_with_an_open_read(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a", "b", "c"], close=False) + async with new_worker(stream_client, ContinueWithOpenReadWorkflow) as worker: + handle = await stream_client.start_workflow( + ContinueWithOpenReadWorkflow.run, + args=[ref.id, None], + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + result = await asyncio.wait_for(handle.result(), timeout=20) + assert result == ["b", "c"] + first_run = stream_client.get_workflow_handle( + handle.id, run_id=handle.first_execution_run_id + ) + await Replayer( + workflows=[ContinueWithOpenReadWorkflow], + stream_config=stream_client.config()["stream_config"], + ).replay_workflow(await first_run.fetch_history()) + + +# A Store written in Python. + + +@workflow.defn +class PythonStoreAppendCloseWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str]) -> None: + stream = workflow.get_stream(ref, body_type=str) + await stream.append("a", "b") + await stream.close() + + +@pytest.mark.parametrize("max_cached_workflows", [1000, 0]) +async def test_python_store_takes_a_workflow_append_and_close( + client: Client, max_cached_workflows: int +): + store = DictStreamStore() + stream_client = _stream_client(client, store) + ref = await _filled_stream(stream_client, [], close=False) + assert ref.store == "dict" + handle, _ = await _run( + stream_client, + PythonStoreAppendCloseWorkflow, + ref, + max_cached_workflows=max_cached_workflows, + ) + assert [record.body for record in await _read_all(stream_client, ref)] == [ + "a", + "b", + ] + assert await _status(stream_client, ref) == StreamStatus.StreamStatusClosed + # The Store gets the Workflow's Producer with the Workflow Task as its epoch. + producer = store.streams[ref.id].records[0].producer + assert producer.WhichOneof("kind") == "workflow" + assert producer.workflow.run_id == handle.result_run_id + assert producer.workflow.epoch.started_event_id > 0 + # Core made the calls, with no Local Activity. + history = await handle.fetch_history() + assert "core_local_activity" not in _markers(history) + assert "core_stream" in _markers(history) + + +@pytest.mark.parametrize("max_cached_workflows", [1000, 0]) +async def test_python_store_gives_a_workflow_the_records_of_a_running_activity( + client: Client, max_cached_workflows: int +): + store = DictStreamStore() + stream_client = _stream_client(client, store) + chunks = await _filled_stream(stream_client, [], close=False) + seen = await _filled_stream(stream_client, [], close=False) + _, text = await _run( + stream_client, + LiveModelWorkflow, + ModelCall(chunks.id, seen.id), + activities=[stream_model_output], + max_cached_workflows=max_cached_workflows, + ) + assert text == "Hello, world" + assert store.waiting == 0 + + +@pytest.mark.parametrize("max_cached_workflows", [1000, 0]) +async def test_python_store_read_with_wait_on_a_quiet_stream_gives_an_empty_page( + client: Client, max_cached_workflows: int +): + store = DictStreamStore() + stream_client = _stream_client(client, store) + ref = await _filled_stream(stream_client, [], close=False) + handle, result = await _run( + stream_client, + ReadOnePageWorkflow, + ref, + 1000, + max_cached_workflows=max_cached_workflows, + ) + assert result == [[], False, None] + events = (await handle.fetch_history()).events + elapsed = events[-1].event_time.ToDatetime() - events[0].event_time.ToDatetime() + assert elapsed >= timedelta(seconds=1) + assert store.waiting == 0 + + +async def test_python_store_read_replays_from_the_store(client: Client): + store = DictStreamStore() + stream_client = _stream_client(client, store) + ref = await _filled_stream(stream_client, ["a", "b", "c"]) + handle, result = await _run( + stream_client, ReadPagesWorkflow, ReadArgs(ref.id, store="dict") + ) + assert result.bodies == ["a", "b", "c"] + # History holds no Record, so the replay reads them from the Python Store again. + recording = RecordingStreamStore(store) + await Replayer( + workflows=[ReadPagesWorkflow], + stream_config=StreamConfig(stores=[recording]), + ).replay_workflow(await handle.fetch_history()) + assert "ReadRecords" in recording.operations + + +class ClosedOnAppendStore(DictStreamStore): + async def append_records( + self, request: AppendRecordsRequest + ) -> AppendRecordsResponse: + raise StreamError( + "The Store closed the Stream", + reason=StreamErrorReason.STREAM_ERROR_REASON_CLOSED, + ) + + +class BrokenReadStore(DictStreamStore): + async def read_records(self, request: ReadRecordsRequest) -> ReadRecordsResponse: + raise RuntimeError("the read broke") + + +@workflow.defn +class PythonStoreErrorWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str], call: str) -> list[Any]: + stream = workflow.get_stream(ref, body_type=str) + try: + if call == "append": + await stream.append("a") + else: + await stream.read_page() + except StreamError as err: + return [StreamErrorReason.Name(err.reason), err.message, err.non_retryable] + return ["no error"] + + +async def test_python_store_error_reaches_workflow_code_with_its_reason( + client: Client, +): + stream_client = _stream_client(client, ClosedOnAppendStore()) + ref = await _filled_stream(stream_client, [], close=False) + _, result = await _run(stream_client, PythonStoreErrorWorkflow, ref, "append") + assert result == [ + "STREAM_ERROR_REASON_CLOSED", + "The Store closed the Stream", + True, + ] + + +class UnavailableTwiceStore(DictStreamStore): + """A Store whose first two appends fail with an outage.""" + + def __init__(self) -> None: + super().__init__() + self.failed_appends = 0 + + async def append_records( + self, request: AppendRecordsRequest + ) -> AppendRecordsResponse: + if self.failed_appends < 2: + self.failed_appends += 1 + raise StreamError( + "The Store is down", + reason=StreamErrorReason.STREAM_ERROR_REASON_STORAGE_UNAVAILABLE, + ) + return await super().append_records(request) + + +async def test_python_store_outage_is_tried_again(client: Client): + store = UnavailableTwiceStore() + stream_client = _stream_client(client, store) + ref = await _filled_stream(stream_client, [], close=False) + _, offsets = await _run(stream_client, AppendWorkflow, ref) + # Core tried the append again after each outage, and the third try wrote it. + assert store.failed_appends == 2 + assert offsets == [(1).to_bytes(8, "big").hex(), (2).to_bytes(8, "big").hex()] + assert [record.body for record in await _read_all(stream_client, ref)] == [ + "a", + "b", + ] + + +class CountingAppendStore(DictStreamStore): + """A Store that allows at most 2 Records in a batch and keeps the size of each append.""" + + def __init__(self) -> None: + super().__init__(limits=StreamBatchLimits(max_batch_records=2)) + self.batches: list[int] = [] + + async def append_records( + self, request: AppendRecordsRequest + ) -> AppendRecordsResponse: + self.batches.append(len(request.records)) + return await super().append_records(request) + + +@workflow.defn +class BatchOverTheLimitWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str]) -> str: + stream = workflow.get_stream(ref, body_type=str) + try: + await stream.append("a", "b", "c") + reason = "appended" + except StreamError as err: + reason = StreamErrorReason.Name(err.reason) + await stream.append("d", "e") + return reason + + +async def test_a_workflow_batch_over_the_limits_of_its_store_is_refused( + client: Client, +): + store = CountingAppendStore() + stream_client = _stream_client(client, store) + ref = await _filled_stream(stream_client, [], close=False) + handle, reason = await _run(stream_client, BatchOverTheLimitWorkflow, ref) + assert reason == "STREAM_ERROR_REASON_TOO_LARGE" + # Core refused the batch of 3 before the Store saw it, and the batch of 2 went through. + assert store.batches == [2] + assert [record.body for record in await _read_all(stream_client, ref)] == [ + "d", + "e", + ] + await Replayer( + workflows=[BatchOverTheLimitWorkflow], + stream_config=stream_client.config()["stream_config"], + ).replay_workflow(await handle.fetch_history()) + assert store.batches == [2] + + +async def test_python_store_exception_is_a_stream_failure_and_the_worker_goes_on( + client: Client, +): + stream_client = _stream_client(client, BrokenReadStore()) + ref = await _filled_stream(stream_client, [], close=False) + async with new_worker( + stream_client, PythonStoreErrorWorkflow, PythonStoreAppendCloseWorkflow + ) as worker: + handle = await stream_client.start_workflow( + PythonStoreErrorWorkflow.run, + args=[ref, "read"], + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + # A retry can't fix the Store, so Core gives the failure on the first try. + result = await asyncio.wait_for(handle.result(), timeout=10) + assert result == [ + "STREAM_ERROR_REASON_STORAGE_CORRUPT", + "The Stream Store 'dict' raised RuntimeError: the read broke", + True, + ] + # The same Worker serves the next Workflow. + await stream_client.execute_workflow( + PythonStoreAppendCloseWorkflow.run, + ref, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + assert await _status(stream_client, ref) == StreamStatus.StreamStatusClosed + + +async def test_python_store_read_that_a_cancel_stops_leaves_no_task(client: Client): + store = DictStreamStore() + stream_client = _stream_client(client, store) + ref = await _filled_stream(stream_client, ["a"], close=False) + async with new_worker(stream_client, CancelDuringReadWorkflow) as worker: + handle = await stream_client.start_workflow( + CancelDuringReadWorkflow.run, + ref.id, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + + async def read_waits() -> None: + assert store.waiting == 1 + + await assert_eventually(read_waits) + await handle.cancel() + assert await asyncio.wait_for(handle.result(), timeout=30) == [ + ["a"], + "cancelled", + ] + + async def no_read_waits() -> None: + assert store.waiting == 0 + + await assert_eventually(no_read_waits) + + +# A payload codec. + + +@workflow.defn +class AppendThenReadWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str]) -> list[str]: + stream = workflow.get_stream(ref, body_type=str) + await stream.append("a", "b") + return [record.body async for record in stream.read()] + + +async def test_payload_codec_covers_direct_append_and_read(client: Client): + store = DictStreamStore() + stream_client = _stream_client(client, store, codec=True) + ref = await _filled_stream(stream_client, [], close=False) + _, bodies = await _run(stream_client, AppendThenReadWorkflow, ref) + assert bodies == ["a", "b"] + # A client without the codec reads the bodies as the Store keeps them. + raw = _stream_client(client, store).get_stream( + StreamRef(id=ref.id), body_type=RawValue + ) + stored = [record.body.payload async for record in raw.read()] + assert [body.metadata["simple-codec"] for body in stored] == [b"true", b"true"] + + +# A close after a Direct append. + + +@workflow.defn +class AppendThenCloseWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str]) -> None: + stream = workflow.get_stream(ref, body_type=str) + await stream.append("a") + await stream.close() + + +async def test_close_after_direct_append(client: Client): + # The append writes as the Workflow with its task as the epoch. The close, from an + # Activity, sends the same Workflow Producer with no epoch, and the Store fences it. + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, [], close=False) + await _run(stream_client, AppendThenCloseWorkflow, ref) + assert [record.body for record in await _read_all(stream_client, ref)] == ["a"] + + +# A measurement. + + +@dataclass +class Burst: + stream_id: str + records: int = 200 + batches: int = 20 + over_ms: int = 2000 + + +@activity.defn +async def write_burst(burst: Burst) -> None: + stream = activity.client().get_stream(StreamRef(id=burst.stream_id), body_type=str) + size = burst.records // burst.batches + for batch in range(burst.batches): + await stream.append( + *(f"record-{batch * size + index}" for index in range(size)) + ) + await asyncio.sleep(burst.over_ms / 1000 / burst.batches) + await stream.close() + + +@workflow.defn +class MeasureWorkflow: + @workflow.run + async def run(self, burst: Burst) -> int: + writer = workflow.start_activity( + write_burst, burst, start_to_close_timeout=timedelta(seconds=60) + ) + stream = workflow.get_stream(StreamRef(id=burst.stream_id), body_type=str) + count = 0 + async for _ in stream.read(wait=timedelta(seconds=30)): + count += 1 + await writer + return count + + +@pytest.mark.parametrize( + "replay_from", [StreamReplaySource.STORE, StreamReplaySource.HISTORY] +) +async def test_measure_read_of_200_records_over_2_seconds( + client: Client, replay_from: StreamReplaySource +): + stream_client = _stream_client(client, replay_from=replay_from) + ref = await _filled_stream(stream_client, [], close=False) + handle, count = await _run( + stream_client, MeasureWorkflow, Burst(ref.id), activities=[write_burst] + ) + assert count == 200 + history = await handle.fetch_history() + sizes = [ + event.marker_recorded_event_attributes.ByteSize() + for event in history.events + if event.event_type == EventType.EVENT_TYPE_MARKER_RECORDED + ] + print( + f"\n{replay_from.name}: {_completed_tasks(history)} Workflow Tasks, " + f"{len(sizes)} markers, {sum(sizes)} marker bytes, sizes {sizes}" + ) + + +# Two loops from one offset, and calls on a read that fails. + + +@workflow.defn +class TwoLoopsWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str]) -> list[list[str]]: + stream = workflow.get_stream(ref, body_type=str) + + async def drain() -> list[str]: + return [record.body async for record in stream.read()] + + # Both loops ask for the first page before either gets it, so each one takes a + # read of its own. + first = asyncio.create_task(drain()) + second = asyncio.create_task(drain()) + return [await first, await second] + + +async def test_two_loops_from_one_offset_each_get_every_record(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a", "b", "c"]) + handle, result = await _run(stream_client, TwoLoopsWorkflow, ref) + assert result == [["a", "b", "c"], ["a", "b", "c"]] + assert len(_read_seqs(await handle.fetch_history())) == 2 + + +@workflow.defn +class FailedReadsWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str]) -> list[str]: + stream = workflow.get_stream(ref, body_type=str) + + async def read() -> str: + try: + await stream.read_page(wait=timedelta(seconds=10)) + except StreamError as err: + return StreamErrorReason.Name(err.reason) + return "page" + + # Two calls wait before the failure comes, and a later call fails too. + first = asyncio.create_task(read()) + second = asyncio.create_task(read()) + return [await first, await second, await read()] + + +async def test_each_read_of_a_missing_stream_raises(client: Client): + stream_client = _stream_client(client) + ref: StreamRef[str] = StreamRef(id=f"stream-{uuid.uuid4()}") + _, result = await _run(stream_client, FailedReadsWorkflow, ref) + assert result == ["STREAM_ERROR_REASON_NOT_FOUND"] * 3 + + +# A codec that uses its serialization context. + + +@dataclass(frozen=True) +class ContextCodec(PayloadCodec, WithSerializationContext): + """Writes the Workflow id of its context into each payload, and decodes only its own.""" + + workflow_id: str = "" + + def with_context(self, context: SerializationContext) -> ContextCodec: + if isinstance(context, WorkflowSerializationContext): + return ContextCodec(context.workflow_id) + return ContextCodec("") + + async def encode(self, payloads: Sequence[Payload]) -> list[Payload]: + return [ + Payload( + metadata={"encoding": b"binary/ctx", "ctx": self.workflow_id.encode()}, + data=payload.SerializeToString(), + ) + for payload in payloads + ] + + async def decode(self, payloads: Sequence[Payload]) -> list[Payload]: + out = [] + for payload in payloads: + if payload.metadata.get("encoding") != b"binary/ctx": + out.append(payload) + continue + if payload.metadata["ctx"] != self.workflow_id.encode(): + raise ApplicationError( + f"Encoded for {payload.metadata['ctx']!r}, not " + f"{self.workflow_id!r}", + non_retryable=True, + ) + out.append(Payload.FromString(payload.data)) + return out + + +@workflow.defn +class AppendThenReadAllWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str]) -> list[str]: + stream = workflow.get_stream(ref, body_type=str) + await stream.append("from-workflow") + return [record.body async for record in stream.read()] + + +async def test_codec_runs_with_no_context_on_stream_bodies(client: Client): + store = DictStreamStore() + config = client.config() + config["stream_config"] = StreamConfig(stores=[store]) + config["data_converter"] = DataConverter(payload_codec=ContextCodec()) + stream_client = Client(**config) + ref = await _filled_stream(stream_client, ["from-client"], close=False) + _, bodies = await _run(stream_client, AppendThenReadAllWorkflow, ref) + assert bodies == ["from-client", "from-workflow"] + # The client decodes with no context, so it reads the Workflow's body too. + assert [record.body for record in await _read_all(stream_client, ref)] == [ + "from-client", + "from-workflow", + ] + raw_config = client.config() + raw_config["stream_config"] = StreamConfig(stores=[store]) + raw = Client(**raw_config).get_stream(StreamRef(id=ref.id), body_type=RawValue) + stored = [record.body.payload async for record in raw.read()] + assert [body.metadata["ctx"] for body in stored] == [b"", b""] + + +@workflow.defn +class AppendOneWorkflow: + @workflow.run + async def run(self, ref: StreamRef[str]) -> None: + await workflow.get_stream(ref, body_type=str).append("from-workflow") + + +# Replays with no Stream config of calls whose ref names no Store. + + +@workflow.defn +class UnnamedRefReadWorkflow: + @workflow.run + async def run(self, stream_id: str) -> list[str]: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + return [record.body async for record in stream.read()] + + +async def _unnamed_read_history( + client: Client, replay_from: StreamReplaySource +) -> tuple[WorkflowHistory, str]: + stream_client = _stream_client(client, replay_from=replay_from) + ref = await _filled_stream(stream_client, ["a", "b"]) + handle, result = await _run(stream_client, UnnamedRefReadWorkflow, ref.id) + assert result == ["a", "b"] + return await handle.fetch_history(), ref.id + + +async def test_history_replay_of_an_unnamed_ref_needs_no_config(client: Client): + history, _ = await _unnamed_read_history(client, StreamReplaySource.HISTORY) + await Replayer(workflows=[UnnamedRefReadWorkflow]).replay_workflow(history) + + +async def test_store_replay_of_an_unnamed_ref_with_no_config_fails(client: Client): + history, stream_id = await _unnamed_read_history(client, StreamReplaySource.STORE) + with pytest.raises(RuntimeError) as err: + await Replayer(workflows=[UnnamedRefReadWorkflow]).replay_workflow(history) + message = str(err.value) + assert f"StreamReplayError: Stream `{stream_id}` in Store `dict`" in message + assert "this Worker has no Stream runtime for the Store `dict`" in message + assert "Nondeterminism" not in message + + +@workflow.defn +class UnnamedRefAppendWorkflow: + @workflow.run + async def run(self, stream_id: str) -> str | None: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + appended = await stream.append("a") + return _hex(appended.last_offset) + + +async def test_replay_of_an_unnamed_append_needs_no_config(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, [], close=False) + handle, _ = await _run(stream_client, UnnamedRefAppendWorkflow, ref.id) + history = await handle.fetch_history() + assert _markers(history) == ["core_stream"] + await Replayer(workflows=[UnnamedRefAppendWorkflow]).replay_workflow(history) + + +# A Record whose body fails to decode. + + +@dataclass +class Point: + x: int + + +@workflow.defn +class BadBodyWorkflow: + @workflow.run + async def run(self, stream_id: str) -> list[Any]: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=Point) + first = await stream.read_page(max_records=1) + events: list[Any] = [["got", first.records[0].body.x]] + # The page of the bad body raises, and a read from the same offset gets it again. + for _ in range(2): + try: + await stream.read_page(from_offset=first.last_offset, max_records=1) + events.append(["page", "decoded"]) + except Exception as err: + events.append(["page", type(err).__name__]) + return events + + +async def test_page_with_a_body_that_fails_to_decode_raises_each_time( + client: Client, +): + stream_client = _stream_client(client) + stream = await stream_client.create_stream(id=f"s-{uuid.uuid4()}", body_type=object) + await stream.append({"x": 1}, "not a point", {"x": 3}) + await stream.close() + _, events = await _run(stream_client, BadBodyWorkflow, stream.ref.id) + assert events == [["got", 1], ["page", "TypeError"], ["page", "TypeError"]] + + +# The Direct close. + + +def _close_entries(history: WorkflowHistory) -> list[StreamCloseEntry]: + """Give the close entries of the `core_stream` markers, in order.""" + entries: list[StreamCloseEntry] = [] + for event in history.events: + attributes = event.marker_recorded_event_attributes + if ( + event.event_type != EventType.EVENT_TYPE_MARKER_RECORDED + or attributes.marker_name != "core_stream" + ): + continue + [payload] = attributes.details["data"].payloads + entries.extend(StreamMarkerData.FromString(payload.data).closes) + return entries + + +def _scheduled_activities(history: WorkflowHistory) -> list[str]: + return [ + event.activity_task_scheduled_event_attributes.activity_type.name + for event in history.events + if event.event_type == EventType.EVENT_TYPE_ACTIVITY_TASK_SCHEDULED + ] + + +async def _status(client: Client, ref: StreamRef[str]) -> StreamStatus | None: + return (await client.get_stream(ref, body_type=str).describe()).status + + +@workflow.defn +class AppendMaybeCloseWorkflow: + @workflow.run + async def run(self, stream_id: str, close: bool) -> None: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + await stream.append("a") + if close: + await stream.close() + + +async def test_awaited_close_closes_in_one_more_task(client: Client): + stream_client = _stream_client(client) + closed_ref = await _filled_stream(stream_client, [], close=False) + open_ref = await _filled_stream(stream_client, [], close=False) + closed, _ = await _run(stream_client, AppendMaybeCloseWorkflow, closed_ref.id, True) + left_open, _ = await _run( + stream_client, AppendMaybeCloseWorkflow, open_ref.id, False + ) + assert await _status(stream_client, closed_ref) == StreamStatus.StreamStatusClosed + assert await _status(stream_client, open_ref) == StreamStatus.StreamStatusOpen + history = await closed.fetch_history() + assert _scheduled_activities(history) == [] + assert "core_local_activity" not in _markers(history) + # The task of the close records its intent, and the next one gives the result. + assert _completed_tasks(history) == ( + _completed_tasks(await left_open.fetch_history()) + 1 + ) + intent, result = _close_entries(history) + assert intent.HasField("intent_epoch") and not intent.resolved + assert result.resolved and not result.HasField("failure") + + +@workflow.defn +class FinallyCloseWorkflow: + @workflow.run + async def run(self, stream_id: str, fail: bool) -> str: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + try: + await stream.append("a") + if fail: + raise ApplicationError("failed on purpose", non_retryable=True) + return "returned" + finally: + await stream.close() + + +@pytest.mark.parametrize("fail", [False, True]) +async def test_close_in_finally(client: Client, fail: bool): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, [], close=False) + if fail: + with pytest.raises(WorkflowFailureError) as err: + await _run(stream_client, FinallyCloseWorkflow, ref.id, True) + assert isinstance(err.value.cause, ApplicationError) + assert err.value.cause.message == "failed on purpose" + else: + _, result = await _run(stream_client, FinallyCloseWorkflow, ref.id, False) + assert result == "returned" + assert await _status(stream_client, ref) == StreamStatus.StreamStatusClosed + assert [record.body for record in await _read_all(stream_client, ref)] == ["a"] + + +@workflow.defn +class AppendAfterCloseWorkflow: + @workflow.run + async def run(self, stream_id: str) -> list[str]: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + await stream.append("a") + await stream.close() + try: + await stream.append("b") + except StreamError as err: + return [StreamErrorReason.Name(err.reason), str(err.non_retryable)] + return ["appended"] + + +async def test_append_after_the_close_is_refused(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, [], close=False) + _, result = await _run(stream_client, AppendAfterCloseWorkflow, ref.id) + assert result == ["STREAM_ERROR_REASON_CLOSED", "True"] + assert [record.body for record in await _read_all(stream_client, ref)] == ["a"] + + +_fail_close_task = True + + +@workflow.defn(sandboxed=False) +class FailAfterCloseOnceWorkflow: + @workflow.run + async def run(self, stream_id: str) -> None: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + await stream.append("a") + close = asyncio.create_task(stream.close()) + # A few turns of the loop let the close send its command in this activation. + for _ in range(10): + await asyncio.sleep(0) + global _fail_close_task + if _fail_close_task: + _fail_close_task = False + raise RuntimeError("Failed the Workflow Task once") + await close + + +async def test_failed_task_after_the_close_leaves_the_stream_open(client: Client): + global _fail_close_task + _fail_close_task = True + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, [], close=False) + handle, _ = await _run(stream_client, FailAfterCloseOnceWorkflow, ref.id) + assert not _fail_close_task + # The retry could append, so the failed task closed nothing. + assert [record.body for record in await _read_all(stream_client, ref)] == ["a", "a"] + assert await _status(stream_client, ref) == StreamStatus.StreamStatusClosed + # History has one intent and one result, from the retry. + entries = _close_entries(await handle.fetch_history()) + assert [entry.resolved for entry in entries] == [False, True] + + +@workflow.defn +class CloseTwiceWorkflow: + @workflow.run + async def run(self, stream_id: str) -> str: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + await stream.close() + await stream.close() + return "closed" + + +async def test_close_of_a_closed_stream_succeeds(client: Client): + stream_client = _stream_client(client) + # A client closed this Stream before the Workflow closes it twice. + ref = await _filled_stream(stream_client, ["a"]) + _, result = await _run(stream_client, CloseTwiceWorkflow, ref.id) + assert result == "closed" + open_ref = await _filled_stream(stream_client, ["a"], close=False) + _, result = await _run(stream_client, CloseTwiceWorkflow, open_ref.id) + assert result == "closed" + assert await _status(stream_client, open_ref) == StreamStatus.StreamStatusClosed + + +async def test_close_without_a_cache_and_in_replays(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, [], close=False) + handle, _ = await _run( + stream_client, AppendMaybeCloseWorkflow, ref.id, True, max_cached_workflows=0 + ) + assert [record.body for record in await _read_all(stream_client, ref)] == ["a"] + assert await _status(stream_client, ref) == StreamStatus.StreamStatusClosed + history = await handle.fetch_history() + assert [entry.resolved for entry in _close_entries(history)] == [False, True] + # A Store with the same Stream, still open, shows that a replay closes nothing. + replay_store = DictStreamStore() + replay_client = _stream_client(client, replay_store) + await replay_client.create_stream(id=ref.id, body_type=str) + await Replayer( + workflows=[AppendMaybeCloseWorkflow], + stream_config=replay_client.config()["stream_config"], + ).replay_workflow(history) + assert await _status(replay_client, ref) == StreamStatus.StreamStatusOpen + # The ref names no Store, so a Replayer with no Stream config takes it from History. + await Replayer(workflows=[AppendMaybeCloseWorkflow]).replay_workflow(history) + + +@workflow.defn +class CloseNotAwaitedWorkflow: + @workflow.run + async def run(self, stream_id: str) -> str: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + # The task sends the close command, and the Workflow returns at once. + asyncio.create_task(stream.close()) + await asyncio.sleep(0) + return "returned" + + +async def test_close_not_awaited_closes_after_the_workflow_ends(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a"], close=False) + handle, result = await _run(stream_client, CloseNotAwaitedWorkflow, ref.id) + assert result == "returned" + + async def closed() -> bool: + return await _status(stream_client, ref) == StreamStatus.StreamStatusClosed + + for _ in range(50): + if await closed(): + break + await asyncio.sleep(0.1) + assert await closed() + # The task that ends the Workflow has the intent, and no task gives a result. + entries = _close_entries(await handle.fetch_history()) + assert [entry.resolved for entry in entries] == [False] + + +async def test_shutdown_waits_for_a_close_not_awaited(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a"], close=False) + async with new_worker(stream_client, CloseNotAwaitedWorkflow) as worker: + result = await stream_client.execute_workflow( + CloseNotAwaitedWorkflow.run, + ref.id, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + assert result == "returned" + # A graceful shutdown returns only after the Store answered the close in flight, so + # the Stream is closed with no wait here. + assert await _status(stream_client, ref) == StreamStatus.StreamStatusClosed + + +@activity.defn +async def sleep_a_while() -> None: + await asyncio.sleep(2) + + +@workflow.defn +class CancelDuringCloseWorkflow: + @workflow.run + async def run(self, stream_id: str) -> str: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + # The Local Activity holds the task of the close open, so the cancel comes in the + # task that gives the result of the close. + workflow.start_local_activity( + sleep_a_while, start_to_close_timeout=timedelta(seconds=10) + ) + try: + await stream.close() + except asyncio.CancelledError: + return "cancelled" + return "closed" + + +async def test_cancel_while_the_close_waits(client: Client): + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a"], close=False) + async with new_worker( + stream_client, CancelDuringCloseWorkflow, activities=[sleep_a_while] + ) as worker: + handle = await stream_client.start_workflow( + CancelDuringCloseWorkflow.run, + ref.id, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + await asyncio.sleep(0.5) + await handle.cancel() + result = await asyncio.wait_for(handle.result(), timeout=30) + # The code gets the cancel, and Core still closed the Stream after the task of the + # intent. + assert result == "cancelled" + assert await _status(stream_client, ref) == StreamStatus.StreamStatusClosed + + +# The metrics of the Direct calls. + + +@workflow.defn +class ReadAppendCloseWorkflow: + @workflow.run + async def run(self, source_id: str, out_id: str) -> list[str]: + source = workflow.get_stream(StreamRef(id=source_id), body_type=str) + out = workflow.get_stream(StreamRef(id=out_id), body_type=str) + got = [record.body async for record in source.read()] + await out.append(*got) + await out.close() + return got + + +async def test_direct_calls_record_their_metrics(env: WorkflowEnvironment): + buffer = MetricBuffer(10000) + runtime = Runtime(telemetry=TelemetryConfig(metrics=buffer)) + client = await env.connect_client(runtime=runtime) + store = DictStreamStore(name="metered") + stream_client = _stream_client(client, store) + source = await _filled_stream(stream_client, ["a", "b", "c"]) + out = await _filled_stream(stream_client, [], close=False) + async with new_worker(stream_client, ReadAppendCloseWorkflow) as worker: + result = await stream_client.execute_workflow( + ReadAppendCloseWorkflow.run, + args=[source.id, out.id], + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + assert result == ["a", "b", "c"] + updates = [ + update + for update in buffer.retrieve_updates() + if update.metric.name.startswith("temporal_stream_") + ] + print("\nStream metrics:", sorted({update.metric.name for update in updates})) + + def values(name: str) -> list[Any]: + return [update.value for update in updates if update.metric.name == name] + + assert sum(values("temporal_stream_read_total")) == 1 + assert sum(values("temporal_stream_read_records")) == 3 + assert sum(values("temporal_stream_append_total")) == 1 + assert sum(values("temporal_stream_close_total")) == 1 + assert len(values("temporal_stream_append_latency")) == 1 + assert len(values("temporal_stream_close_latency")) == 1 + assert values("temporal_stream_read_failed") == [] + assert values("temporal_stream_append_failed") == [] + assert values("temporal_stream_close_failed") == [] + for update in updates: + # The Stream runtime counts the reads that wait on the Store, with the default + # attributes of the meter and no more. + if update.metric.name == "temporal_stream_blocking_reads_waiting": + continue + assert update.attributes["stream_store"] == "metered" + assert update.attributes["workflow_type"] == "ReadAppendCloseWorkflow" + assert update.attributes["task_queue"] == worker.task_queue + + +# The example of the README: a Workflow starts each tool as its call arrives. + + +@dataclass +class ToolCall: + name: str + argument: str + + +@dataclass +class ModelEvent: + kind: str + text: str | None = None + tool_call: ToolCall | None = None + + +_MODEL_EVENTS = [ + ModelEvent(kind="text", text="Let me look."), + ModelEvent(kind="tool_call", tool_call=ToolCall(name="weather", argument="Oslo")), + ModelEvent(kind="text", text="And the time."), + ModelEvent(kind="tool_call", tool_call=ToolCall(name="clock", argument="Oslo")), + ModelEvent(kind="text", text="Done."), +] + + +@activity.defn +async def model_answer(events_id: str) -> None: + events = activity.client().get_stream(StreamRef(id=events_id), body_type=ModelEvent) + for event in _MODEL_EVENTS: + await events.append(event) + # The model takes a while between its events, so a tool can start before it ends. + await asyncio.sleep(0.5) + await events.close() + + +@activity.defn +async def run_tool(call: ToolCall) -> str: + return f"{call.name}({call.argument})" + + +@workflow.defn +class ToolCallsWorkflow: + @workflow.run + async def run(self, events_id: str, status_id: str) -> list[str]: + model = workflow.start_activity( + model_answer, events_id, start_to_close_timeout=timedelta(seconds=30) + ) + events = workflow.get_stream(StreamRef(id=events_id), body_type=ModelEvent) + status = workflow.get_stream(StreamRef(id=status_id), body_type=str) + tools = [] + async for record in events.read(wait=timedelta(seconds=10)): + event = record.body + if event.kind == "tool_call": + assert event.tool_call is not None + # The tool starts now, before the model ends its answer. + tools.append( + workflow.start_activity( + run_tool, + event.tool_call, + start_to_close_timeout=timedelta(minutes=1), + ) + ) + await status.append(f"Running {event.tool_call.name}") + results = await asyncio.gather(*tools) + await model + return list(results) + + +async def test_workflow_starts_each_tool_as_its_call_arrives(client: Client): + stream_client = _stream_client(client) + events = await _filled_stream(stream_client, [], close=False) + status = await _filled_stream(stream_client, [], close=False) + handle, results = await _run( + stream_client, + ToolCallsWorkflow, + events.id, + status.id, + activities=[model_answer, run_tool], + ) + assert results == ["weather(Oslo)", "clock(Oslo)"] + assert [record.body for record in await _read_all(stream_client, status)] == [ + "Running weather", + "Running clock", + ] + # The first tool was scheduled while the model Activity still ran. + history = await handle.fetch_history() + scheduled: dict[int, str] = {} + first_tool = None + model_completed = None + for event in history.events: + if event.event_type == EventType.EVENT_TYPE_ACTIVITY_TASK_SCHEDULED: + name = event.activity_task_scheduled_event_attributes.activity_type.name + scheduled[event.event_id] = name + if name == "run_tool" and first_tool is None: + first_tool = event.event_id + elif event.event_type == EventType.EVENT_TYPE_ACTIVITY_TASK_COMPLETED: + attributes = event.activity_task_completed_event_attributes + if scheduled[attributes.scheduled_event_id] == "model_answer": + model_completed = event.event_id + assert first_tool is not None and model_completed is not None + assert first_tool < model_completed + + +# The read-ahead bound. + + +def test_read_ahead_bytes_below_one_raises(): + for bytes_ in (0, -1): + with pytest.raises(ValueError, match="workflow_read_ahead_bytes"): + StreamConfig(stores=[DictStreamStore()], workflow_read_ahead_bytes=bytes_) + + +async def test_read_with_a_read_ahead_of_one_byte_gives_every_record(client: Client): + config = client.config() + config["stream_config"] = StreamConfig( + stores=[DictStreamStore()], workflow_read_ahead_bytes=1 + ) + stream_client = Client(**config) + bodies = [f"record-{index}" for index in range(20)] + ref = await _filled_stream(stream_client, bodies) + _, result = await _run(stream_client, ReadPagesWorkflow, ReadArgs(ref.id)) + assert result.bodies == bodies + assert result.dones[-1] is True + + +# The cap on the Records of one page that Workflow code gets. + + +@pytest.mark.parametrize("max_cached_workflows", [1000, 0]) +async def test_store_page_over_256_records_comes_in_pages_of_at_most_256( + client: Client, max_cached_workflows: int +): + stream_client = _stream_client(client) + bodies = [f"record-{index}" for index in range(600)] + ref = await _filled_stream(stream_client, bodies) + handle, got = await _run( + stream_client, + ReadItemsWorkflow, + ref, + None, + max_cached_workflows=max_cached_workflows, + ) + assert got == bodies + history = await handle.fetch_history() + counts = [ + count + for task in _read_entries(history) + for entry in task + for count in entry.page_counts + ] + assert sum(counts) == 600 + assert max(counts) == 256 + config = stream_client.config()["stream_config"] + await Replayer(workflows=[ReadItemsWorkflow], stream_config=config).replay_workflow( + history + ) + + +# The page cap and the least gap of a read, two settings of the Worker. + + +def _client_with(client: Client, **settings: Any) -> Client: + config = client.config() + config["stream_config"] = StreamConfig(stores=[DictStreamStore()], **settings) + return Client(**config) + + +def _page_counts(history: WorkflowHistory) -> list[int]: + return [ + count + for task in _read_entries(history) + for entry in task + for count in entry.page_counts + ] + + +@pytest.mark.parametrize("max_cached_workflows", [1000, 0]) +async def test_page_cap_of_10_gives_pages_of_at_most_10( + client: Client, max_cached_workflows: int +): + stream_client = _client_with(client, workflow_read_page_records=10) + bodies = [f"record-{index}" for index in range(35)] + ref = await _filled_stream(stream_client, bodies) + handle, got = await _run( + stream_client, + ReadItemsWorkflow, + ref, + None, + max_cached_workflows=max_cached_workflows, + ) + assert got == bodies + history = await handle.fetch_history() + counts = _page_counts(history) + assert sum(counts) == 35 + assert max(counts) == 10 + config = stream_client.config()["stream_config"] + await Replayer(workflows=[ReadItemsWorkflow], stream_config=config).replay_workflow( + history + ) + + +def test_page_cap_out_of_bounds_raises(): + for records in (0, 1001): + with pytest.raises(ValueError, match="workflow_read_page_records"): + StreamConfig(stores=[DictStreamStore()], workflow_read_page_records=records) + + +def test_min_gap_out_of_bounds_raises(): + for gap in (timedelta(seconds=2), timedelta(milliseconds=-1)): + with pytest.raises(ValueError, match="workflow_read_min_gap"): + StreamConfig(stores=[DictStreamStore()], workflow_read_min_gap=gap) + + +@activity.defn +async def append_five_20ms_apart(stream_id: str) -> None: + stream = activity.client().get_stream(StreamRef(id=stream_id), body_type=str) + for index in range(5): + await stream.append(f"record-{index}") + await asyncio.sleep(0.02) + await stream.close() + + +@workflow.defn +class GappedReadWorkflow: + @workflow.run + async def run(self, stream_id: str) -> ReadResult: + writer = workflow.start_activity( + append_five_20ms_apart, + stream_id, + start_to_close_timeout=timedelta(seconds=30), + ) + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + result = ReadResult([], [], [], [], []) + async for page in stream.read_pages(wait=timedelta(seconds=10)): + result.bodies.extend(record.body for record in page.records) + result.page_sizes.append(len(page.records)) + result.dones.append(page.done) + await writer + return result + + +async def test_min_gap_of_200ms_joins_records_that_come_20ms_apart(client: Client): + stream_client = _client_with( + client, workflow_read_min_gap=timedelta(milliseconds=200) + ) + ref = await _filled_stream(stream_client, [], close=False) + handle, result = await _run( + stream_client, + GappedReadWorkflow, + ref.id, + activities=[append_five_20ms_apart], + ) + assert result.bodies == [f"record-{index}" for index in range(5)] + assert len(result.page_sizes) < 5 + assert result.dones[-1] is True + config = stream_client.config()["stream_config"] + await Replayer( + workflows=[GappedReadWorkflow], stream_config=config + ).replay_workflow(await handle.fetch_history()) + + +# A replay that lost the Records of a read. + + +class _ReadThenWait: + def __init__(self) -> None: + self.go = False + + async def read_then_wait(self, stream_id: str) -> list[str]: + stream = workflow.get_stream(StreamRef(id=stream_id), body_type=str) + got = [record.body async for record in stream.read()] + await workflow.wait_condition(lambda: self.go) + return got + + @workflow.signal + def proceed(self) -> None: + self.go = True + + +@workflow.defn +class ReadThenWaitWorkflow(_ReadThenWait): + @workflow.run + async def run(self, stream_id: str) -> list[str]: + return await self.read_then_wait(stream_id) + + +@workflow.defn(failure_exception_types=[StreamReplayError]) +class ReadThenWaitFailsWorkflow(_ReadThenWait): + @workflow.run + async def run(self, stream_id: str) -> list[str]: + return await self.read_then_wait(stream_id) + + +async def _replay_with_records_gone( + client: Client, workflow_type: type[Any], **worker_options: Any +) -> WorkflowHandle[Any, Any]: + """Read on one Worker, then give the run to a Worker whose Store lost the Records.""" + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a", "b"]) + task_queue = f"tq-{uuid.uuid4()}" + # No cache, so the next Worker replays the History from the start. + async with new_worker( + stream_client, workflow_type, task_queue=task_queue, max_cached_workflows=0 + ): + handle: WorkflowHandle[Any, Any] = await stream_client.start_workflow( + workflow_type.run, + ref.id, + id=f"workflow-{uuid.uuid4()}", + task_queue=task_queue, + ) + + async def read_is_in_history() -> None: + assert "core_stream" in _markers(await handle.fetch_history()) + + await assert_eventually(read_is_in_history) + # A new Store of the same name has none of the Records. + lost_client = _stream_client(client) + worker = new_worker( + lost_client, workflow_type, task_queue=task_queue, **worker_options + ) + async with worker: + await handle.signal("proceed") + if workflow_type is ReadThenWaitWorkflow and not worker_options: + await assert_task_fail_eventually( + handle, message_contains=f"StreamReplayError: Stream `{ref.id}`" + ) + await handle.terminate() + return handle + with pytest.raises(WorkflowFailureError) as err: + await asyncio.wait_for(handle.result(), timeout=30) + cause = err.value.cause + assert isinstance(cause, ApplicationError) + assert cause.type == "StreamReplayError" + assert f"StreamReplayError: Stream `{ref.id}`" in cause.message + return handle + + +async def test_lost_records_fail_the_task_by_default(client: Client): + await _replay_with_records_gone(client, ReadThenWaitWorkflow) + + +async def test_lost_records_fail_the_workflow_with_the_worker_option(client: Client): + await _replay_with_records_gone( + client, + ReadThenWaitWorkflow, + workflow_failure_exception_types=[StreamReplayError], + ) + + +async def test_lost_records_fail_the_workflow_with_the_defn_option(client: Client): + await _replay_with_records_gone(client, ReadThenWaitFailsWorkflow) + + +# A Workflow outbound interceptor sees each page that Workflow code reads. + +_intercepted: list[tuple[str, Any]] = [] + + +class StreamCallInterceptor(Interceptor): + def workflow_interceptor_class( + self, input: WorkflowInterceptorClassInput + ) -> type[WorkflowInboundInterceptor]: + return _StreamCallInbound + + +class _StreamCallInbound(WorkflowInboundInterceptor): + def init(self, outbound: WorkflowOutboundInterceptor) -> None: + super().init(_StreamCallOutbound(outbound)) + + +class _StreamCallOutbound(WorkflowOutboundInterceptor): + async def start_read_records( + self, + request: stream_service.ReadRecordsRequest[Any], + output_type: type[stream_service.ReadRecordsResponse[Any]] | None = None, + ) -> workflow.NexusOperationHandle[stream_service.ReadRecordsResponse[Any]]: + _intercepted.append(("ReadRecords", request)) + return await super().start_read_records(request, output_type) + + +async def test_workflow_interceptor_sees_each_page_read(client: Client): + _intercepted.clear() + stream_client = _stream_client(client) + ref = await _filled_stream(stream_client, ["a", "b", "c", "d", "e"]) + async with new_worker( + stream_client, ReadPagesWorkflow, interceptors=[StreamCallInterceptor()] + ) as worker: + result = await stream_client.execute_workflow( + ReadPagesWorkflow.run, + ReadArgs(ref.id, max_records=2), + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + assert result.page_sizes == [2, 2, 1] + assert [name for name, _ in _intercepted] == ["ReadRecords"] * 3 + # Each page starts after the last Record of the page before it. + assert [request.from_offset for _, request in _intercepted] == [ + None, + bytes.fromhex(result.offsets[1]), + bytes.fromhex(result.offsets[3]), + ] diff --git a/tests/nexus/test_stream_store.py b/tests/nexus/test_stream_store.py index f9a582519f..fa7aa0af00 100644 --- a/tests/nexus/test_stream_store.py +++ b/tests/nexus/test_stream_store.py @@ -20,6 +20,7 @@ import temporalio.client._stream_store import temporalio.common import temporalio.converter._payload_converter +import temporalio.exceptions import temporalio.nexus.system import temporalio.nexus.system._support import temporalio.workflow @@ -33,7 +34,7 @@ ) from temporalio.client._stream_store import _STREAM_OPERATIONS, _request_ref from temporalio.converter import DataConverter, StorageDriverActivityInfo -from temporalio.exceptions import StreamConfigError, StreamError +from temporalio.exceptions import ApplicationError, StreamConfigError, StreamError from temporalio.nexus.system.stream_service import ( ClientProducer, DescribeStreamRequest, @@ -662,3 +663,32 @@ def test_stream_error_carries_its_reason(): assert StreamError("x").reason == ( StreamErrorReason.STREAM_ERROR_REASON_UNSPECIFIED ) + + +def test_stream_error_from_application_error(): + from_failure = temporalio.exceptions._stream_error_from_failure + rebuilt = from_failure( + ApplicationError( + "the Stream exists", + temporalio.api.stream.v1.StreamErrorDetails( + reason=StreamErrorReason.STREAM_ERROR_REASON_ALREADY_EXISTS + ), + type="StreamError", + ) + ) + assert isinstance(rebuilt, StreamError) + assert rebuilt.reason == StreamErrorReason.STREAM_ERROR_REASON_ALREADY_EXISTS + assert rebuilt.message == "the Stream exists" + unknown = from_failure( + ApplicationError( + "x", + temporalio.api.stream.v1.StreamErrorDetails( + reason=StreamErrorReason.ValueType(999) + ), + type="StreamError", + ) + ) + assert unknown is not None + assert unknown.reason == StreamErrorReason.STREAM_ERROR_REASON_UNSPECIFIED + assert from_failure(ApplicationError("x", type="RuntimeError")) is None + assert from_failure(RuntimeError("x")) is None diff --git a/tests/nexus/test_stream_store_workflow.py b/tests/nexus/test_stream_store_workflow.py index c1df44daaa..01984d4604 100644 --- a/tests/nexus/test_stream_store_workflow.py +++ b/tests/nexus/test_stream_store_workflow.py @@ -1,84 +1,250 @@ from __future__ import annotations +import asyncio import uuid +from collections.abc import Callable, Sequence from datetime import timedelta +from typing import Any import pytest +from google.protobuf.message import Message +import temporalio.common +import temporalio.exceptions from temporalio import activity, workflow -from temporalio.client import Client +from temporalio.api.enums.v1 import EventType, StreamErrorReason +from temporalio.api.streamservice.v1 import ( + CreateStreamRequest as CreateStreamRequestMessage, +) +from temporalio.api.streamservice.v1 import ( + CreateStreamResponse as CreateStreamResponseMessage, +) +from temporalio.api.streamservice.v1 import ( + DescribeStreamRequest as DescribeStreamRequestMessage, +) +from temporalio.api.streamservice.v1 import ( + DescribeStreamResponse as DescribeStreamResponseMessage, +) +from temporalio.client import ( + Client, + StreamConfig, + StreamSelectContext, + StreamStore, + StreamStoreCallOptions, + WorkflowFailureError, + WorkflowHandle, +) +from temporalio.common import RetryPolicy +from temporalio.converter import DataConverter, StorageDriverWorkflowInfo +from temporalio.exceptions import ApplicationError, StreamError from temporalio.nexus.system import TEMPORAL_SYSTEM_ENDPOINT from temporalio.nexus.system.stream_service import ( + AppendRecordsResponse, + ClientProducer, Producer, + ProducerKindClient, ProducerKindLocalActivity, + ProducerKindWorkflow, ReadRecordsRequest, + StreamIdConflictPolicy, StreamRecord, StreamRef, StreamService, + StreamStatus, ) from temporalio.nexus.system.stream_service.workflow import create_stream, get_stream +from temporalio.nexus.system.stream_service.workflow._resources.stream import ( + Stream as WorkflowStream, +) +from temporalio.worker import ( + ActivityInboundInterceptor, + ExecuteActivityInput, + Interceptor, + Replayer, + Worker, +) from tests.helpers import assert_task_fail_eventually, new_worker from tests.helpers.streams import DictStreamStore, RecordingStreamStore -from tests.nexus.test_stream_store import _with_store +from tests.test_converter import SimpleCodec +# Workflow code writes these values, so the History shows only whether a Record body +# leaks into it. _BODIES = ["first-record-body", "second-record-body"] +@activity.defn +async def append_values(ref: StreamRef[str], values: list[str]) -> None: + await activity.client().get_stream(ref, body_type=str).append(*values) + + +async def _append_in_local_activity(ref: StreamRef[str], values: list[str]) -> None: + await workflow.execute_local_activity( + append_values, args=[ref, values], start_to_close_timeout=timedelta(seconds=10) + ) + + @workflow.defn -class StreamCallWorkflow: +class AppendWorkflow: @workflow.run - async def run(self, operation: str) -> None: - stream = get_stream(StreamRef(id="stream"), body_type=str) - if operation == "CreateStream": - await create_stream(id="stream", body_type=str) - elif operation == "AppendRecords": + async def run(self, stream_id: str) -> StreamRef[str]: + stream = await create_stream(id=stream_id, body_type=str) + await stream.append(*_BODIES) + return stream.ref + + +@workflow.defn +class LocalActivityAppendWorkflow: + @workflow.run + async def run(self, stream_id: str) -> StreamRef[str]: + stream = await create_stream(id=stream_id, body_type=str) + await _append_in_local_activity(stream.ref, _BODIES) + return stream.ref + + +# Workflow code appends either with the Workflow `Stream` or with a Local Activity of +# its own. Core makes the first append in the Workflow Task as the Workflow, and the +# second writes as the Local Activity that runs it. +_APPEND_WORKFLOWS = pytest.mark.parametrize( + "append_workflow", [AppendWorkflow, LocalActivityAppendWorkflow] +) + + +@workflow.defn +class AppendOffsetsWorkflow: + @workflow.run + async def run(self, stream_id: str) -> AppendRecordsResponse: + stream = await create_stream(id=stream_id, body_type=str) + return await stream.append(*_BODIES) + + +@workflow.defn +class AppendToStreamWorkflow: + @workflow.run + async def run(self, stream_id: str) -> str: + stream = get_stream(StreamRef(id=stream_id), body_type=str) + try: await stream.append("a") - elif operation == "CloseStream": - await stream.close() - elif operation == "DescribeStream": - await stream.describe() - else: - # The Workflow `Stream` has no `read`, but a Nexus client can start one. - nexus_client = workflow.create_nexus_client( - service=StreamService, endpoint=TEMPORAL_SYSTEM_ENDPOINT - ) - await nexus_client.start_operation( - operation, ReadRecordsRequest(ref=StreamRef[str](id="stream")) + except StreamError as err: + assert isinstance(err, ApplicationError) + return StreamErrorReason.Name(err.reason) + return "appended" + + +@workflow.defn +class ExplicitProducerWorkflow: + @workflow.run + async def run(self, stream_id: str) -> str: + stream = await create_stream(id=stream_id, body_type=str) + try: + await stream.append( + "a", + producer=Producer(id="mine", kind=ProducerKindClient(ClientProducer())), ) + except StreamError as err: + return f"{StreamErrorReason.Name(err.reason)}: {err.message}" + return "appended" -@pytest.mark.parametrize( - "operation", - ["CreateStream", "AppendRecords", "CloseStream", "DescribeStream", "ReadRecords"], -) -async def test_workflow_stream_call_fails_the_workflow_task( - client: Client, operation: str -): - store = RecordingStreamStore(DictStreamStore()) - stream_client = _with_store(client, store) - async with new_worker(stream_client, StreamCallWorkflow) as worker: - handle = await stream_client.start_workflow( - StreamCallWorkflow.run, - operation, - id=f"workflow-{uuid.uuid4()}", - task_queue=worker.task_queue, - execution_timeout=timedelta(seconds=30), - ) - await assert_task_fail_eventually( - handle, - message_contains=( - "Workflow code can't call a Stream in this version. Call it from an " - "Activity through activity.client()." - ), +@workflow.defn +class NexusClientReadWorkflow: + @workflow.run + async def run(self, operation: str) -> str: + # Workflow code can also start the operation of `read_page` through a Nexus client. + requests: dict[str, Any] = { + "ReadRecords": ReadRecordsRequest(ref=StreamRef[str](id="stream")), + } + nexus_client = workflow.create_nexus_client( + service=StreamService, endpoint=TEMPORAL_SYSTEM_ENDPOINT ) - await handle.terminate() - # The Store of the Worker's client gets no call. - assert store.operations == [] + try: + handle: workflow.NexusOperationHandle[ + Any + ] = await nexus_client.start_operation(operation, requests[operation]) + await handle + except StreamError as err: + return StreamErrorReason.Name(err.reason) + except ApplicationError as err: + assert err.non_retryable + return err.message + return "read" + + +@workflow.defn +class DescribeResultWorkflow: + @workflow.run + async def run(self, stream_id: str) -> str: + stream = await create_stream(id=stream_id, body_type=str) + try: + await stream.describe() + except StreamError as err: + return f"StreamError {StreamErrorReason.Name(err.reason)}: {err.message}" + except ApplicationError as err: + return f"ApplicationError {err.type}: {err.message}" + except temporalio.exceptions.TimeoutError as err: + return f"TimeoutError {err.type and err.type.name}" + return "described" + + +@workflow.defn +class DescribeStreamWorkflow: + @workflow.run + async def run(self, stream_id: str) -> str: + stream = get_stream(StreamRef(id=stream_id), body_type=str) + try: + await stream.describe() + except StreamError as err: + assert isinstance(err, ApplicationError) + return StreamErrorReason.Name(err.reason) + return "described" + + +@workflow.defn +class CreateThenContinueAsNewWorkflow: + @workflow.run + async def run(self, stream_id: str, run: int) -> list[bool]: + stream = await create_stream(id=stream_id, body_type=str) + await _append_in_local_activity(stream.ref, [f"run-{run}"]) + if run == 1: + workflow.continue_as_new(args=[stream_id, 2]) + return [bool(stream.created)] + + +@workflow.defn +class DescribeOtherStoreWorkflow: + @workflow.run + async def run(self, stream_id: str) -> None: + await get_stream( + StreamRef(id=stream_id, store="other"), body_type=str + ).describe() + + +@workflow.defn +class CreateAndDescribeWorkflow: + @workflow.run + async def run(self, stream_id: str) -> StreamRef[str]: + stream = await create_stream(id=stream_id, body_type=str) + await stream.describe() + return stream.ref + + +class CountingStore(RecordingStreamStore): + def __init__(self, name: str) -> None: + super().__init__(DictStreamStore(name=name)) @activity.defn -async def append_values(ref: StreamRef[str], values: list[str]) -> None: - await activity.client().get_stream(ref, body_type=str).append(*values) +async def user_activity() -> None: + pass + + +@workflow.defn +class CreateAndActivityWorkflow: + @workflow.run + async def run(self, stream_id: str) -> None: + await create_stream(id=stream_id, body_type=str) + await workflow.execute_local_activity( + user_activity, start_to_close_timeout=timedelta(seconds=10) + ) _fail_next_task = True @@ -87,12 +253,16 @@ async def append_values(ref: StreamRef[str], values: list[str]) -> None: @workflow.defn(sandboxed=False) class AppendThenFailTaskOnceWorkflow: @workflow.run - async def run(self, ref: StreamRef[str]) -> None: - await workflow.execute_local_activity( - append_values, - args=[ref, _BODIES], - start_to_close_timeout=timedelta(seconds=10), + async def run(self, stream_id: str, in_local_activity: bool) -> None: + stream = await create_stream( + id=stream_id, + body_type=str, + id_conflict_policy=StreamIdConflictPolicy.StreamIdConflictPolicyUseExisting, ) + if in_local_activity: + await _append_in_local_activity(stream.ref, _BODIES) + else: + await stream.append(*_BODIES) global _fail_next_task if _fail_next_task: _fail_next_task = False @@ -116,13 +286,145 @@ async def run(self, stream_id: str) -> None: ) +class FlakyDescribeStore(RecordingStreamStore): + def __init__(self, *, failures: int = 0, delay: timedelta = timedelta()) -> None: + super().__init__(DictStreamStore()) + self._failures = failures + self._delay = delay + self.describes = 0 + + async def before_call(self, operation: str, request: Message) -> None: + if operation == "DescribeStream": + self.describes += 1 + await asyncio.sleep(self._delay.total_seconds()) + if self.describes <= self._failures: + raise RuntimeError("describe failed") + + +class FlakyCreateStore(CountingStore): + def __init__(self, name: str) -> None: + super().__init__(name) + self.failed_creates = 0 + + async def before_call(self, operation: str, request: Message) -> None: + if operation == "CreateStream" and not self.failed_creates: + self.failed_creates += 1 + raise RuntimeError("create failed") + + +@workflow.defn +class CloseWorkflow: + @workflow.run + async def run(self, stream_id: str) -> StreamRef[str]: + stream = await create_stream(id=stream_id, body_type=str) + await stream.append("a") + await stream.close() + return stream.ref + + +@workflow.defn +class AsyncWithCloseWorkflow: + @workflow.run + async def run(self, stream_id: str) -> str: + async with create_stream(id=stream_id, body_type=str) as stream: + await stream.append("a") + described = await stream.describe() + assert described.status is not None + return described.status.name + + +_fail_close_intent_task = True + + +@workflow.defn(sandboxed=False) +class CloseIntentThenFailTaskOnceWorkflow: + @workflow.run + async def run(self, stream_id: str) -> None: + stream = get_stream(StreamRef(id=stream_id), body_type=str) + await stream.append("a") + close = asyncio.create_task(stream.close()) + # A few turns of the loop let the close send its intent in this task. + for _ in range(10): + await asyncio.sleep(0) + global _fail_close_intent_task + if _fail_close_intent_task: + _fail_close_intent_task = False + raise RuntimeError("Failed the Workflow Task once") + await close + + +class RecordingInterceptor(Interceptor): + def __init__(self) -> None: + self.activity_types: list[str] = [] + self.is_local: list[bool] = [] + + def intercept_activity( + self, next: ActivityInboundInterceptor + ) -> ActivityInboundInterceptor: + return _RecordingActivityInbound(next, self) + + +class _RecordingActivityInbound(ActivityInboundInterceptor): + def __init__( + self, next: ActivityInboundInterceptor, recording: RecordingInterceptor + ) -> None: + super().__init__(next) + self._recording = recording + + async def execute_activity(self, input: ExecuteActivityInput) -> Any: + info = activity.info() + self._recording.activity_types.append(info.activity_type) + self._recording.is_local.append(info.is_local) + return await super().execute_activity(input) + + +def _with_store(client: Client, store: StreamStore) -> Client: + return _with_config(client, StreamConfig(stores=[store])) + + +def _with_config(client: Client, stream_config: StreamConfig) -> Client: + config = client.config() + config["stream_config"] = stream_config + return Client(**config) + + +def _two_counting_stores( + contexts: list[StreamSelectContext], +) -> tuple[CountingStore, CountingStore, StreamConfig]: + first = CountingStore("first") + second = CountingStore("second") + + def select(context: StreamSelectContext) -> StreamStore: + contexts.append(context) + return second + + return first, second, StreamConfig(stores=[first, second], store_selector=select) + + +async def _run_create_and_describe( + client: Client, stream_id: str +) -> tuple[WorkflowHandle[CreateAndDescribeWorkflow, StreamRef[str]], StreamRef[str]]: + async with new_worker(client, CreateAndDescribeWorkflow) as worker: + handle = await client.start_workflow( + CreateAndDescribeWorkflow.run, + stream_id, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + return handle, await handle.result() + + async def _read_all(client: Client, stream_id: str) -> list[StreamRecord[str]]: stream = client.get_stream(StreamRef(id=stream_id), body_type=str) return [record async for record in stream.read()] -def _local_activity_epoch(producer: Producer) -> tuple[int, int, int]: - # The Workflow Task, then the attempt of the Local Activity. +def _epoch(producer: Producer) -> tuple[int, int, int]: + # The Workflow Task, then the attempt of the Local Activity, which a Workflow lacks. + if isinstance(producer.kind, ProducerKindWorkflow): + task = producer.kind.value.epoch + assert task is not None + return (task.started_event_id, task.attempt, 0) assert isinstance(producer.kind, ProducerKindLocalActivity) local = producer.kind.value.epoch return (local.task.started_event_id, local.task.attempt, local.attempt) @@ -134,33 +436,346 @@ def _producers(records: list[StreamRecord[str]]) -> list[Producer]: return [producer for producer in producers if producer is not None] -async def test_local_activity_append_after_a_failed_workflow_task_has_a_higher_epoch( +async def _run_append_workflow( + client: Client, append_workflow: type[Any] = AppendWorkflow +) -> tuple[WorkflowHandle[Any, StreamRef[str]], StreamRef[str]]: + async with new_worker( + client, append_workflow, activities=[append_values] + ) as worker: + handle: WorkflowHandle[Any, StreamRef[str]] = await client.start_workflow( + append_workflow.run, + f"stream-{uuid.uuid4()}", + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + return handle, await handle.result() + + +async def _describe_result( + client: Client, + store: StreamStore, + options: StreamStoreCallOptions | None = None, +) -> str: + stream_config = StreamConfig( + stores=[store], workflow_call_options=options or StreamStoreCallOptions() + ) + stream_client = _with_config(client, stream_config) + async with new_worker(stream_client, DescribeResultWorkflow) as worker: + return await stream_client.execute_workflow( + DescribeResultWorkflow.run, + f"stream-{uuid.uuid4()}", + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + + +@_APPEND_WORKFLOWS +async def test_workflow_stream_uses_store(client: Client, append_workflow: type[Any]): + stream_client = _with_store(client, DictStreamStore()) + handle, ref = await _run_append_workflow(stream_client, append_workflow) + + stream = stream_client.get_stream(ref, body_type=str) + records = [record async for record in stream.read()] + assert [record.body for record in records] == _BODIES + run_id = handle.result_run_id + assert run_id + first, second = _producers(records) + assert first == second + if append_workflow is AppendWorkflow: + # The Workflow writes as itself, with its first Workflow Task as the epoch. + assert first.id == f"wf:{run_id}" + assert isinstance(first.kind, ProducerKindWorkflow) + assert first.kind.value.run_id == run_id + assert first.kind.value.epoch is not None + assert first.kind.value.epoch.started_event_id > 0 + assert first.kind.value.epoch.attempt == 1 + return + # The create is the run's first Local Activity, so the append is its second. + assert first.id == f"la:{run_id}:2" + assert isinstance(first.kind, ProducerKindLocalActivity) + assert first.kind.value.run_id == run_id + assert first.kind.value.epoch.task.started_event_id > 0 + assert first.kind.value.epoch.attempt == 1 + + +async def test_workflow_stream_uses_a_store_written_from_scratch(client: Client): + store = DictStreamStore() + stream_client = _with_store(client, store) + stream_id = f"stream-{uuid.uuid4()}" + _, ref = await _run_create_and_describe(stream_client, stream_id) + assert ref == StreamRef(id=stream_id, store="dict") + assert list(store.streams) == [stream_id] + + stream = stream_client.get_stream(ref, body_type=str) + await stream.append("a") + assert [record.body async for record in stream.read()] == ["a"] + + +@_APPEND_WORKFLOWS +async def test_workflow_stream_store_uses_payload_codec( + client: Client, append_workflow: type[Any] +): + store = DictStreamStore() + config = client.config() + config["data_converter"] = DataConverter(payload_codec=SimpleCodec()) + config["stream_config"] = StreamConfig(stores=[store]) + stream_client = Client(**config) + _, ref = await _run_append_workflow(stream_client, append_workflow) + + # A client without the codec reads the bodies as the Store keeps them. + raw = _with_store(client, store).get_stream( + StreamRef(id=ref.id), body_type=temporalio.common.RawValue + ) + stored = [record.body.payload async for record in raw.read()] + assert [body.metadata["simple-codec"] for body in stored] == [b"true", b"true"] + stream = stream_client.get_stream(ref, body_type=str) + assert [record.body async for record in stream.read()] == _BODIES + + +@_APPEND_WORKFLOWS +async def test_workflow_stream_history_has_no_bodies( + client: Client, append_workflow: type[Any] +): + stream_client = _with_store(client, DictStreamStore()) + handle, _ = await _run_append_workflow(stream_client, append_workflow) + events = (await handle.fetch_history()).events + markers = [ + event.marker_recorded_event_attributes.marker_name + for event in events + if event.event_type == EventType.EVENT_TYPE_MARKER_RECORDED + ] + # One marker for the create and one for the append. Core writes its own marker for + # an append that it makes in the Workflow Task. + append_marker = ( + "core_stream" if append_workflow is AppendWorkflow else "core_local_activity" + ) + assert markers == ["core_local_activity", append_marker] + for event in events: + for body in _BODIES: + assert body.encode() not in event.SerializeToString() + + +@_APPEND_WORKFLOWS +async def test_workflow_stream_replay_does_not_call_store( + client: Client, append_workflow: type[Any] +): + stream_client = _with_store(client, DictStreamStore()) + handle, ref = await _run_append_workflow(stream_client, append_workflow) + # A replay of an append that Core made needs a Store, as the first run had. A new + # Store shows any call, since it has no Stream. + replay_store = DictStreamStore() + await Replayer( + workflows=[append_workflow], + stream_config=StreamConfig(stores=[replay_store]), + ).replay_workflow(await handle.fetch_history()) + with pytest.raises(StreamError) as err: + await ( + _with_store(client, replay_store).get_stream(ref, body_type=str).describe() + ) + assert err.value.reason == StreamErrorReason.STREAM_ERROR_REASON_NOT_FOUND + + +async def test_workflow_create_without_stream_config_fails_the_workflow( client: Client, +): + async with new_worker(client, CreateAndDescribeWorkflow) as worker: + handle = await client.start_workflow( + CreateAndDescribeWorkflow.run, + f"stream-{uuid.uuid4()}", + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + execution_timeout=timedelta(seconds=30), + ) + with pytest.raises(WorkflowFailureError) as err: + await asyncio.wait_for(handle.result(), timeout=10) + cause = err.value.cause + assert isinstance(cause, ApplicationError) + assert not isinstance(cause, StreamError) + assert cause.type == "StreamConfigError" + assert cause.non_retryable + assert cause.message == ( + "The client of this Worker has no Stream Store. Give one in " + "stream_config=StreamConfig(stores=[...])." + ) + # A Replayer with no Stream config fails the Workflow the same way. + await Replayer(workflows=[CreateAndDescribeWorkflow]).replay_workflow( + await handle.fetch_history() + ) + + +async def test_workflow_read_through_a_nexus_client_goes_through_core( + client: Client, +): + stream_client = _with_store(client, DictStreamStore()) + async with new_worker(stream_client, NexusClientReadWorkflow) as worker: + result = await stream_client.execute_workflow( + NexusClientReadWorkflow.run, + "ReadRecords", + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + # Core reads the Store, as for `read_page`. Nobody created the Stream. + assert result == "STREAM_ERROR_REASON_NOT_FOUND" + + +def test_workflow_stream_has_the_calls_of_a_client_stream(): + for name in ( + "append", + "close", + "describe", + "read", + "read_pages", + "read_page", + "__aenter__", + "__aexit__", + ): + assert hasattr(WorkflowStream, name) + assert not hasattr(WorkflowStream, "iter") + + +async def test_workflow_stream_store_error_reaches_workflow_code(client: Client): + store = FlakyDescribeStore(failures=1) + result = await _describe_result(client, store) + assert result == "ApplicationError RuntimeError: describe failed" + assert store.describes == 1 + + +async def test_workflow_describe_of_unknown_stream_raises_stream_error(client: Client): + stream_client = _with_store(client, DictStreamStore()) + async with new_worker(stream_client, DescribeStreamWorkflow) as worker: + reason = await stream_client.execute_workflow( + DescribeStreamWorkflow.run, + f"stream-{uuid.uuid4()}", + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + assert reason == "STREAM_ERROR_REASON_NOT_FOUND" + + +async def test_strict_create_in_each_run_of_a_chain_succeeds(client: Client): + stream_client = _with_store(client, DictStreamStore()) + stream_id = f"stream-{uuid.uuid4()}" + async with new_worker( + stream_client, CreateThenContinueAsNewWorkflow, activities=[append_values] + ) as worker: + handle = await stream_client.start_workflow( + CreateThenContinueAsNewWorkflow.run, + args=[stream_id, 1], + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + assert await handle.result() == [True] + first_run_id = handle.first_execution_run_id + last_run = await stream_client.get_workflow_handle(handle.id).describe() + assert first_run_id + assert last_run.run_id != first_run_id + records = await _read_all(stream_client, stream_id) + assert [record.body for record in records] == ["run-1", "run-2"] + first, second = _producers(records) + assert isinstance(first.kind, ProducerKindLocalActivity) + assert isinstance(second.kind, ProducerKindLocalActivity) + assert first.kind.value.run_id == first_run_id + assert second.kind.value.run_id == last_run.run_id + + +@_APPEND_WORKFLOWS +async def test_client_append_after_workflow_append_keeps_offset_order( + client: Client, append_workflow: type[Any] +): + stream_client = _with_store(client, DictStreamStore()) + _, ref = await _run_append_workflow(stream_client, append_workflow) + stream = stream_client.get_stream(ref, body_type=str) + appended = await stream.append("from-client") + records = await _read_all(stream_client, ref.id) + assert [record.body for record in records] == [*_BODIES, "from-client"] + offsets = [record.offset for record in records] + assert offsets == sorted(offsets) + assert len(set(offsets)) == len(offsets) + assert offsets[-1] == appended.last_offset + kinds = [type(producer.kind) for producer in _producers(records)] + workflow_kind = ( + ProducerKindWorkflow + if append_workflow is AppendWorkflow + else ProducerKindLocalActivity + ) + assert kinds == [workflow_kind, workflow_kind, ProducerKindClient] + + +async def test_workflow_stream_store_call_options_retry(client: Client): + options = StreamStoreCallOptions( + retry_policy=RetryPolicy( + initial_interval=timedelta(milliseconds=10), maximum_attempts=2 + ) + ) + store = FlakyDescribeStore(failures=1) + assert await _describe_result(client, store, options) == "described" + assert store.describes == 2 + + +async def test_workflow_stream_store_call_options_timeout(client: Client): + options = StreamStoreCallOptions(start_to_close_timeout=timedelta(milliseconds=200)) + store = FlakyDescribeStore(delay=timedelta(seconds=5)) + result = await _describe_result(client, store, options) + assert result == "TimeoutError START_TO_CLOSE" + + +async def test_worker_keeps_its_stream_stores(client: Client): + stream_client = _with_store(client, DictStreamStore()) + async with new_worker(stream_client, AppendWorkflow) as worker: + with pytest.raises(ValueError, match="Stream Stores"): + worker.client = client + worker.client = stream_client + + +async def test_activity_interceptors_see_stream_store_activity(client: Client): + interceptor = RecordingInterceptor() + stream_client = _with_store(client, DictStreamStore()) + async with new_worker( + stream_client, + CreateAndActivityWorkflow, + activities=[user_activity], + interceptors=[interceptor], + ) as worker: + await stream_client.execute_workflow( + CreateAndActivityWorkflow.run, + f"stream-{uuid.uuid4()}", + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + # The create runs as the SDK's Local Activity, before the Workflow's own Activity. + assert interceptor.activity_types == [ + "__temporal_stream_store_call:dict", + "user_activity", + ] + + +@pytest.mark.parametrize("in_local_activity", [False, True]) +async def test_workflow_task_retry_appends_with_higher_epoch( + client: Client, in_local_activity: bool ): global _fail_next_task _fail_next_task = True stream_client = _with_store(client, DictStreamStore()) - stream = await stream_client.create_stream( - id=f"stream-{uuid.uuid4()}", body_type=str - ) + stream_id = f"stream-{uuid.uuid4()}" async with new_worker( stream_client, AppendThenFailTaskOnceWorkflow, activities=[append_values] ) as worker: await stream_client.execute_workflow( AppendThenFailTaskOnceWorkflow.run, - stream.ref, + args=[stream_id, in_local_activity], id=f"workflow-{uuid.uuid4()}", task_queue=worker.task_queue, ) - records = await _read_all(stream_client, stream.ref.id) - # The failed Workflow Task records no marker, so its retry runs the Local Activity - # again and writes a second copy. + records = await _read_all(stream_client, stream_id) + # The failed Workflow Task records no marker, so its retry appends again and writes + # a second copy. assert [record.body for record in records] == [*_BODIES, *_BODIES] producers = _producers(records) assert len({producer.id for producer in producers}) == 1 - first, second, third, fourth = ( - _local_activity_epoch(producer) for producer in producers - ) + if not in_local_activity: + assert producers[0].id.startswith("wf:") + + first, second, third, fourth = (_epoch(producer) for producer in producers) assert first == second assert third == fourth assert third > first @@ -190,3 +805,390 @@ async def test_user_local_activity_append_has_local_activity_producer(client: Cl assert producer.kind.value.run_id == run_id assert producer.kind.value.epoch.task.started_event_id > 0 assert producer.kind.value.epoch.attempt == 1 + + +async def test_workflow_ref_of_another_store_fails_the_task(client: Client): + stream_client = _with_store(client, DictStreamStore()) + async with new_worker(stream_client, DescribeOtherStoreWorkflow) as worker: + handle = await stream_client.start_workflow( + DescribeOtherStoreWorkflow.run, + f"stream-{uuid.uuid4()}", + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + execution_timeout=timedelta(seconds=30), + ) + await assert_task_fail_eventually( + handle, message_contains="names the Stream Store 'other'" + ) + await handle.terminate() + + +async def test_workflow_create_without_store_on_several_stores_fails_the_task( + client: Client, +): + stream_config = StreamConfig( + stores=[CountingStore("first"), CountingStore("second")] + ) + stream_client = _with_config(client, stream_config) + stream_id = f"stream-{uuid.uuid4()}" + async with new_worker(stream_client, CreateAndDescribeWorkflow) as worker: + handle = await stream_client.start_workflow( + CreateAndDescribeWorkflow.run, + stream_id, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + execution_timeout=timedelta(seconds=30), + ) + await assert_task_fail_eventually( + handle, + message_contains=( + "StreamConfig has more than one Store and no store_selector, and the " + f"call on the Stream '{stream_id}' names no Store. Name one with " + "store=, or set StreamConfig.store_selector." + ), + ) + await handle.terminate() + + +async def test_workflow_create_goes_to_the_selected_store(client: Client): + contexts: list[StreamSelectContext] = [] + first, second, stream_config = _two_counting_stores(contexts) + stream_id = f"stream-{uuid.uuid4()}" + handle, ref = await _run_create_and_describe( + _with_config(client, stream_config), stream_id + ) + assert ref.store == "second" + assert first.operations == [] + assert second.operations == ["CreateStream", "DescribeStream"] + # The describe goes by the Store that the ref names, so the selector runs once. + assert contexts == [ + StreamSelectContext( + id=stream_id, + target=StorageDriverWorkflowInfo( + id=handle.id, + type="CreateAndDescribeWorkflow", + run_id=handle.result_run_id, + namespace=client.namespace, + ), + ) + ] + + +async def test_workflow_replay_calls_no_store(client: Client): + _, _, stream_config = _two_counting_stores([]) + stream_id = f"stream-{uuid.uuid4()}" + handle, _ = await _run_create_and_describe( + _with_config(client, stream_config), stream_id + ) + contexts: list[StreamSelectContext] = [] + first, second, replay_config = _two_counting_stores(contexts) + await Replayer( + workflows=[CreateAndDescribeWorkflow], stream_config=replay_config + ).replay_workflow(await handle.fetch_history()) + # The selector runs in Workflow code, so a replay runs it again with the Workflow. + assert [context.id for context in contexts] == [stream_id] + target = contexts[0].target + assert isinstance(target, StorageDriverWorkflowInfo) + assert (target.id, target.run_id) == (handle.id, handle.result_run_id) + assert first.operations == [] + assert second.operations == [] + + +async def test_workflow_create_with_several_stores_takes_the_config_options( + client: Client, +): + first = CountingStore("first") + second = FlakyCreateStore("second") + stream_config = StreamConfig( + stores=[first, second], + store_selector=lambda _: second, + workflow_call_options=StreamStoreCallOptions( + retry_policy=RetryPolicy( + initial_interval=timedelta(milliseconds=10), maximum_attempts=2 + ) + ), + ) + # The default options make one attempt, so only the config's retry policy lets + # the create pass after its first attempt fails. + _, ref = await _run_create_and_describe( + _with_config(client, stream_config), f"stream-{uuid.uuid4()}" + ) + assert ref.store == "second" + assert first.operations == [] + assert second.operations == ["CreateStream", "CreateStream", "DescribeStream"] + + +class OwnActivityStore(CountingStore): + """A Store that brings its own Activity for the calls of Workflow code.""" + + def __init__(self) -> None: + super().__init__("own") + self.workflow_calls: list[str] = [] + + def worker_activities(self) -> Sequence[Callable[..., Any]]: + @activity.defn(name="own_store_call") + async def own_store_call(operation: str, request: bytes) -> bytes: + if operation == "CreateStream": + created = await self.create_stream( + CreateStreamRequestMessage.FromString(request) + ) + return created.SerializeToString() + described = await self.describe_stream( + DescribeStreamRequestMessage.FromString(request) + ) + return described.SerializeToString() + + return [own_store_call] + + async def call_from_workflow( + self, operation: str, request: Message, options: StreamStoreCallOptions + ) -> Message: + self.workflow_calls.append(operation) + response = await workflow.execute_local_activity( + "own_store_call", + args=[operation, request.SerializeToString()], + result_type=bytes, + start_to_close_timeout=options.start_to_close_timeout, + ) + if operation == "CreateStream": + return CreateStreamResponseMessage.FromString(response) + return DescribeStreamResponseMessage.FromString(response) + + +async def test_workflow_call_goes_through_the_store_own_activity(client: Client): + store = OwnActivityStore() + interceptor = RecordingInterceptor() + stream_client = _with_store(client, store) + stream_id = f"stream-{uuid.uuid4()}" + async with new_worker( + stream_client, CreateAndDescribeWorkflow, interceptors=[interceptor] + ) as worker: + ref = await stream_client.execute_workflow( + CreateAndDescribeWorkflow.run, + stream_id, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + assert ref == StreamRef(id=stream_id, store="own") + assert store.workflow_calls == ["CreateStream", "DescribeStream"] + assert store.operations == ["CreateStream", "DescribeStream"] + assert interceptor.activity_types == ["own_store_call", "own_store_call"] + + +async def test_workflow_append_returns_offsets_of_records_in_order(client: Client): + stream_client = _with_store(client, DictStreamStore()) + stream_id = f"stream-{uuid.uuid4()}" + async with new_worker(stream_client, AppendOffsetsWorkflow) as worker: + appended = await stream_client.execute_workflow( + AppendOffsetsWorkflow.run, + stream_id, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + records = await _read_all(stream_client, stream_id) + assert [record.body for record in records] == _BODIES + assert [appended.first_offset, appended.last_offset] == [ + record.offset for record in records + ] + + +async def test_workflow_append_to_closed_stream_raises_stream_error(client: Client): + stream_client = _with_store(client, DictStreamStore()) + stream = await stream_client.create_stream( + id=f"stream-{uuid.uuid4()}", body_type=str + ) + await stream.close() + async with new_worker(stream_client, AppendToStreamWorkflow) as worker: + reason = await stream_client.execute_workflow( + AppendToStreamWorkflow.run, + stream.ref.id, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + assert reason == "STREAM_ERROR_REASON_CLOSED" + + +async def test_workflow_append_with_a_producer_of_another_kind_fails(client: Client): + stream_client = _with_store(client, DictStreamStore()) + stream_id = f"stream-{uuid.uuid4()}" + async with new_worker(stream_client, ExplicitProducerWorkflow) as worker: + result = await stream_client.execute_workflow( + ExplicitProducerWorkflow.run, + stream_id, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + # Core writes only as the Workflow's Producer, so the append writes nothing. + assert result == ( + "STREAM_ERROR_REASON_INVALID: A Stream append of Workflow code writes as a " + "Producer of kind Workflow. Leave out producer=, or give a Producer of kind " + "Workflow." + ) + assert await _read_all(stream_client, stream_id) == [] + + +async def _describe_status(client: Client, stream_id: str) -> StreamStatus | None: + stream = client.get_stream(StreamRef(id=stream_id), body_type=str) + return (await stream.describe()).status + + +async def test_workflow_async_with_closes_the_stream(client: Client): + stream_client = _with_store(client, DictStreamStore()) + stream_id = f"stream-{uuid.uuid4()}" + async with new_worker(stream_client, AsyncWithCloseWorkflow) as worker: + status = await stream_client.execute_workflow( + AsyncWithCloseWorkflow.run, + stream_id, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + assert status == StreamStatus.StreamStatusOpen.name + assert await _describe_status(stream_client, stream_id) == ( + StreamStatus.StreamStatusClosed + ) + records = await _read_all(stream_client, stream_id) + assert [record.body for record in records] == ["a"] + + +@pytest.mark.parametrize("no_remote_activities", [False, True]) +async def test_worker_with_workflows_only_runs_the_close( + client: Client, no_remote_activities: bool +): + stream_client = _with_store(client, DictStreamStore()) + stream_id = f"stream-{uuid.uuid4()}" + async with Worker( + stream_client, + task_queue=str(uuid.uuid4()), + workflows=[CloseWorkflow], + no_remote_activities=no_remote_activities, + ) as worker: + await stream_client.execute_workflow( + CloseWorkflow.run, + stream_id, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + assert await _describe_status(stream_client, stream_id) == ( + StreamStatus.StreamStatusClosed + ) + + +async def test_failed_workflow_task_after_direct_close_leaves_the_stream_open( + client: Client, +): + global _fail_close_intent_task + _fail_close_intent_task = True + stream_client = _with_store(client, DictStreamStore()) + stream = await stream_client.create_stream( + id=f"stream-{uuid.uuid4()}", body_type=str + ) + async with new_worker(stream_client, CloseIntentThenFailTaskOnceWorkflow) as worker: + await stream_client.execute_workflow( + CloseIntentThenFailTaskOnceWorkflow.run, + stream.ref.id, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + # Core makes the close with no Activity. The failed task recorded no intent, so it + # closed nothing, and the retried task appends again before it closes the Stream. + records = await _read_all(stream_client, stream.ref.id) + assert [record.body for record in records] == ["a", "a"] + assert await _describe_status(stream_client, stream.ref.id) == ( + StreamStatus.StreamStatusClosed + ) + + +@workflow.defn +class TwoAppendsWorkflow: + @workflow.run + async def run(self, stream_id: str, sleep_between: bool) -> None: + stream = await create_stream(id=stream_id, body_type=str) + await stream.append("a") + if sleep_between: + # A timer ends the Workflow Task, so the next append runs in a later one. + await workflow.sleep(timedelta(milliseconds=10)) + await stream.append("b") + + +@workflow.defn +class GatheredAppendsWorkflow: + @workflow.run + async def run(self, stream_id: str) -> None: + stream = await create_stream(id=stream_id, body_type=str) + await asyncio.gather(stream.append("a"), stream.append("b"), stream.append("c")) + + +@workflow.defn +class GatheredAppendAndCloseWorkflow: + @workflow.run + async def run(self, stream_id: str) -> None: + stream = await create_stream(id=stream_id, body_type=str) + await asyncio.gather(stream.append("a"), stream.close()) + + +async def _run_two_appends( + client: Client, store: StreamStore, sleep_between: bool +) -> tuple[str, list[StreamRecord[str]]]: + stream_client = _with_store(client, store) + stream_id = f"stream-{uuid.uuid4()}" + async with new_worker(stream_client, TwoAppendsWorkflow) as worker: + handle = await stream_client.start_workflow( + TwoAppendsWorkflow.run, + args=[stream_id, sleep_between], + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + await handle.result() + assert handle.result_run_id + return handle.result_run_id, await _read_all(stream_client, stream_id) + + +async def test_workflow_appends_in_one_task_write_as_one_producer(client: Client): + run_id, records = await _run_two_appends( + client, DictStreamStore(), sleep_between=False + ) + assert [record.body for record in records] == ["a", "b"] + first, second = _producers(records) + assert first == second + assert first.id == f"wf:{run_id}" + + +async def test_workflow_append_in_a_later_task_has_a_higher_epoch(client: Client): + run_id, records = await _run_two_appends( + client, DictStreamStore(), sleep_between=True + ) + first, second = _producers(records) + assert first.id == second.id == f"wf:{run_id}" + assert _epoch(second) > _epoch(first) + + +async def test_workflow_appends_started_together_write_in_call_order(client: Client): + stream_client = _with_store(client, DictStreamStore()) + stream_id = f"stream-{uuid.uuid4()}" + async with new_worker(stream_client, GatheredAppendsWorkflow) as worker: + await stream_client.execute_workflow( + GatheredAppendsWorkflow.run, + stream_id, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + # Core runs the appends of one Stream one at a time, in the order of the calls. + records = await _read_all(stream_client, stream_id) + assert [record.body for record in records] == ["a", "b", "c"] + + +async def test_workflow_close_started_with_an_append_comes_after_it(client: Client): + stream_client = _with_store(client, DictStreamStore()) + stream_id = f"stream-{uuid.uuid4()}" + async with new_worker(stream_client, GatheredAppendAndCloseWorkflow) as worker: + await stream_client.execute_workflow( + GatheredAppendAndCloseWorkflow.run, + stream_id, + id=f"workflow-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + records = await _read_all(stream_client, stream_id) + assert [record.body for record in records] == ["a"] + assert await _describe_status(stream_client, stream_id) == ( + StreamStatus.StreamStatusClosed + )