From 132621c38dddfb91c103e71fa554f42e5f58e67d Mon Sep 17 00:00:00 2001 From: jmaeagle99 <44687433+jmaeagle99@users.noreply.github.com> Date: Tue, 10 Nov 2026 15:25:43 -0800 Subject: [PATCH 1/3] Add external storage sample --- README.md | 2 + core/build.gradle | 5 + .../samples/externalstorage/BatchSummary.java | 3 + .../ExternalStorageWorker.java | 32 +++++ .../samples/externalstorage/Order.java | 3 + .../externalstorage/OrderActivities.java | 12 ++ .../externalstorage/OrderActivitiesImpl.java | 33 +++++ .../ProcessOrderBatchWorkflow.java | 15 +++ .../ProcessOrderBatchWorkflowImpl.java | 20 +++ .../samples/externalstorage/README.md | 120 ++++++++++++++++++ .../samples/externalstorage/S3Mock.java | 33 +++++ .../samples/externalstorage/Starter.java | 42 ++++++ .../externalstorage/StorageConfig.java | 57 +++++++++ core/src/main/resources/logback.xml | 1 + 14 files changed, 378 insertions(+) create mode 100644 core/src/main/java/io/temporal/samples/externalstorage/BatchSummary.java create mode 100644 core/src/main/java/io/temporal/samples/externalstorage/ExternalStorageWorker.java create mode 100644 core/src/main/java/io/temporal/samples/externalstorage/Order.java create mode 100644 core/src/main/java/io/temporal/samples/externalstorage/OrderActivities.java create mode 100644 core/src/main/java/io/temporal/samples/externalstorage/OrderActivitiesImpl.java create mode 100644 core/src/main/java/io/temporal/samples/externalstorage/ProcessOrderBatchWorkflow.java create mode 100644 core/src/main/java/io/temporal/samples/externalstorage/ProcessOrderBatchWorkflowImpl.java create mode 100644 core/src/main/java/io/temporal/samples/externalstorage/README.md create mode 100644 core/src/main/java/io/temporal/samples/externalstorage/S3Mock.java create mode 100644 core/src/main/java/io/temporal/samples/externalstorage/Starter.java create mode 100644 core/src/main/java/io/temporal/samples/externalstorage/StorageConfig.java diff --git a/README.md b/README.md index b70a90983..2b4147987 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,8 @@ See the README.md file in each main sample directory for cut/paste Gradle comman - [**Worker Versioning**](/core/src/main/java/io/temporal/samples/workerversioning): Demonstrates how to use worker versioning to manage workflow code changes. +- [**External Storage**](/core/src/main/java/io/temporal/samples/externalstorage): Demonstrates offloading large payloads to S3 with a compression codec and a mock S3 server. + - [**Environment Configuration**](/core/src/main/java/io/temporal/samples/envconfig): Load client configuration from TOML files with programmatic overrides. diff --git a/core/build.gradle b/core/build.gradle index a06ff7f56..d2e6939e1 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -43,6 +43,11 @@ dependencies { implementation("software.amazon.awssdk:kms") implementation("software.amazon.awssdk:dynamodb") + // Used in External Storage sample + implementation "io.temporal:temporal-payload-storage-s3driver-awssdkv2:$javaSDKVersion" + implementation "software.amazon.awssdk:s3" + implementation "org.gaul:s3proxy:4.1.1" + // we don't update it to 2.1.0 because 2.1.0 requires Java 11 implementation 'com.codingrodent:jackson-json-crypto:1.1.0' diff --git a/core/src/main/java/io/temporal/samples/externalstorage/BatchSummary.java b/core/src/main/java/io/temporal/samples/externalstorage/BatchSummary.java new file mode 100644 index 000000000..49e54bd06 --- /dev/null +++ b/core/src/main/java/io/temporal/samples/externalstorage/BatchSummary.java @@ -0,0 +1,3 @@ +package io.temporal.samples.externalstorage; + +public record BatchSummary(int orderCount, double totalWeightKg) {} diff --git a/core/src/main/java/io/temporal/samples/externalstorage/ExternalStorageWorker.java b/core/src/main/java/io/temporal/samples/externalstorage/ExternalStorageWorker.java new file mode 100644 index 000000000..0300f137c --- /dev/null +++ b/core/src/main/java/io/temporal/samples/externalstorage/ExternalStorageWorker.java @@ -0,0 +1,32 @@ +package io.temporal.samples.externalstorage; + +import io.temporal.client.WorkflowClient; +import io.temporal.envconfig.ClientConfigProfile; +import io.temporal.serviceclient.WorkflowServiceStubs; +import io.temporal.worker.Worker; +import io.temporal.worker.WorkerFactory; +import java.io.IOException; + +public class ExternalStorageWorker { + + static final String TASK_QUEUE = "external-storage"; + + public static void main(String[] args) throws IOException { + ClientConfigProfile profile = ClientConfigProfile.load(); + WorkflowServiceStubs service = + WorkflowServiceStubs.newServiceStubs(profile.toWorkflowServiceStubsOptions()); + // The worker uses its client's data converter and external storage, so this is all it takes to + // offload the large activity inputs and results it sends. + WorkflowClient client = + WorkflowClient.newInstance( + service, StorageConfig.withExternalStorage(profile.toWorkflowClientOptions())); + + WorkerFactory factory = WorkerFactory.newInstance(client); + Worker worker = factory.newWorker(TASK_QUEUE); + worker.registerWorkflowImplementationTypes(ProcessOrderBatchWorkflowImpl.class); + worker.registerActivitiesImplementations(new OrderActivitiesImpl()); + + factory.start(); + System.out.println("Worker started on task queue: " + TASK_QUEUE); + } +} diff --git a/core/src/main/java/io/temporal/samples/externalstorage/Order.java b/core/src/main/java/io/temporal/samples/externalstorage/Order.java new file mode 100644 index 000000000..b0900cfb5 --- /dev/null +++ b/core/src/main/java/io/temporal/samples/externalstorage/Order.java @@ -0,0 +1,3 @@ +package io.temporal.samples.externalstorage; + +public record Order(String id, double weightKg, String notes) {} diff --git a/core/src/main/java/io/temporal/samples/externalstorage/OrderActivities.java b/core/src/main/java/io/temporal/samples/externalstorage/OrderActivities.java new file mode 100644 index 000000000..bfb486e06 --- /dev/null +++ b/core/src/main/java/io/temporal/samples/externalstorage/OrderActivities.java @@ -0,0 +1,12 @@ +package io.temporal.samples.externalstorage; + +import io.temporal.activity.ActivityInterface; +import java.util.List; + +@ActivityInterface +public interface OrderActivities { + + List fetchOrders(int orderCount); + + BatchSummary summarizeOrders(List orders); +} diff --git a/core/src/main/java/io/temporal/samples/externalstorage/OrderActivitiesImpl.java b/core/src/main/java/io/temporal/samples/externalstorage/OrderActivitiesImpl.java new file mode 100644 index 000000000..942e46786 --- /dev/null +++ b/core/src/main/java/io/temporal/samples/externalstorage/OrderActivitiesImpl.java @@ -0,0 +1,33 @@ +package io.temporal.samples.externalstorage; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +public class OrderActivitiesImpl implements OrderActivities { + + /** + * Random notes don't compress well, so the order list stays above the external storage threshold + * after the codec compresses it. + */ + private static final int NOTES_UUIDS_PER_ORDER = 100; + + @Override + public List fetchOrders(int orderCount) { + List orders = new ArrayList<>(orderCount); + for (int i = 1; i <= orderCount; i++) { + StringBuilder notes = new StringBuilder(); + for (int j = 0; j < NOTES_UUIDS_PER_ORDER; j++) { + notes.append(UUID.randomUUID()); + } + orders.add(new Order(String.format("ORD-%06d", i), i % 50 + 0.5, notes.toString())); + } + return orders; + } + + @Override + public BatchSummary summarizeOrders(List orders) { + double totalWeightKg = orders.stream().mapToDouble(Order::weightKg).sum(); + return new BatchSummary(orders.size(), totalWeightKg); + } +} diff --git a/core/src/main/java/io/temporal/samples/externalstorage/ProcessOrderBatchWorkflow.java b/core/src/main/java/io/temporal/samples/externalstorage/ProcessOrderBatchWorkflow.java new file mode 100644 index 000000000..142abb6ff --- /dev/null +++ b/core/src/main/java/io/temporal/samples/externalstorage/ProcessOrderBatchWorkflow.java @@ -0,0 +1,15 @@ +package io.temporal.samples.externalstorage; + +import io.temporal.workflow.WorkflowInterface; +import io.temporal.workflow.WorkflowMethod; + +/** + * The workflow's own input and result are small. The order list passed from {@code fetchOrders} to + * {@code summarizeOrders} is the payload large enough to be offloaded to external storage. + */ +@WorkflowInterface +public interface ProcessOrderBatchWorkflow { + + @WorkflowMethod + BatchSummary run(int orderCount); +} diff --git a/core/src/main/java/io/temporal/samples/externalstorage/ProcessOrderBatchWorkflowImpl.java b/core/src/main/java/io/temporal/samples/externalstorage/ProcessOrderBatchWorkflowImpl.java new file mode 100644 index 000000000..8bcbc3691 --- /dev/null +++ b/core/src/main/java/io/temporal/samples/externalstorage/ProcessOrderBatchWorkflowImpl.java @@ -0,0 +1,20 @@ +package io.temporal.samples.externalstorage; + +import io.temporal.activity.ActivityOptions; +import io.temporal.workflow.Workflow; +import java.time.Duration; +import java.util.List; + +public class ProcessOrderBatchWorkflowImpl implements ProcessOrderBatchWorkflow { + + private final OrderActivities activities = + Workflow.newActivityStub( + OrderActivities.class, + ActivityOptions.newBuilder().setStartToCloseTimeout(Duration.ofSeconds(60)).build()); + + @Override + public BatchSummary run(int orderCount) { + List orders = activities.fetchOrders(orderCount); + return activities.summarizeOrders(orders); + } +} diff --git a/core/src/main/java/io/temporal/samples/externalstorage/README.md b/core/src/main/java/io/temporal/samples/externalstorage/README.md new file mode 100644 index 000000000..67eac9142 --- /dev/null +++ b/core/src/main/java/io/temporal/samples/externalstorage/README.md @@ -0,0 +1,120 @@ +# External Storage + +> **Note:** External storage is experimental and may change in the future. + +This sample offloads large payloads to S3-compatible object storage with the SDK's external storage +support and the `temporal-payload-storage-s3driver-awssdkv2` module. The SDK's `ZlibPayloadCodec` +compresses every payload first, so what is stored inline in Temporal and in S3 is compressed. + +**Scenario:** The workflow receives an order count and calls a `fetchOrders` activity that returns +that many orders. The list, about 400 KiB even after compression, is passed to a second +`summarizeOrders` activity, and the workflow returns its small `BatchSummary`. + +The codec compresses each payload, and the SDK then compares the compressed size against the default +256 KiB threshold. Payloads still above it are stored in S3 and replaced in history by a small +reference. The workflow's own input and result are a few bytes and stay inline. + +A mock S3 server ([S3Mock.java](S3Mock.java)), built on [S3Proxy](https://github.com/gaul/s3proxy), +is included, so the sample runs locally without an AWS account or Docker. + +## Running this sample + +1. `temporal server start-dev` to start the Temporal server. +2. In another terminal, start the mock S3 server. It listens on `127.0.0.1:5000`, serves the + `temporal-payloads` bucket, and keeps objects in memory until it stops. + + ```bash + ./gradlew -q :core:execute -PmainClass=io.temporal.samples.externalstorage.S3Mock + ``` + +3. In another terminal, start the worker: + + ```bash + ./gradlew -q :core:execute -PmainClass=io.temporal.samples.externalstorage.ExternalStorageWorker + ``` + +4. In another terminal, run the workflow: + + ```bash + ./gradlew -q :core:execute -PmainClass=io.temporal.samples.externalstorage.Starter + ``` + + Example output: + + ```text + Started workflow, run ID: + Processed 200 orders weighing 5000.0 kg in total + ``` + +5. Run `temporal workflow show` to see how the payloads were stored: + + ```bash + temporal workflow show --workflow-id external-storage-workflow-id + ``` + + | Payload | Type | Storage | + | ------------------------ | ------------------------ | ----------------- | + | Workflow input | `int` | Inline | + | `fetchOrders` input | `int` | Inline | + | `fetchOrders` result | `List` (~400 KiB) | **External (S3)** | + | `summarizeOrders` input | `List` (~400 KiB) | **External (S3)** | + | `summarizeOrders` result | `BatchSummary` | Inline | + | Workflow result | `BatchSummary` | Inline | + + The two offloaded payloads are the order list passed from one activity to the next, which is + what external storage is for: large data that flows between activities without bloating + workflow history. Both references name the same S3 object: the driver keys objects by a hash of + their content, so when the workflow passes the list on, it finds the object already stored and + skips the upload. + +## How it works + +[StorageConfig.java](StorageConfig.java) adds the codec and external storage to the +`WorkflowClientOptions` that both the worker and the starter use. The worker uses its client's +configuration, so it needs no other setup. + +```java +S3StorageDriver driver = + S3StorageDriver.newBuilder() + .setClient(new S3AsyncClientAdapter(s3Client)) + .setBucket("temporal-payloads") + .build(); + +WorkflowClientOptions.newBuilder(options) + .setDataConverter( + new CodecDataConverter( + DefaultDataConverter.newDefaultInstance(), + Collections.singletonList(new ZlibPayloadCodec()))) + .setExternalStorage(ExternalStorage.newBuilder().setDriver(driver).build()) + .build(); +``` + +When encoding a payload, the SDK: + +1. Serializes the value to a `Payload`. +2. Runs the codec, which compresses the payload. +3. Compares the compressed size against the threshold (default 256 KiB, set with + `ExternalStorage.Builder.setPayloadSizeThreshold`). +4. If it is at or above the threshold, stores the compressed payload in S3 and replaces it with a + reference. + +Decoding reverses these steps, retrieving from S3 and then decompressing. + +Every process that reads or writes this workflow's payloads must use the same codec and external +storage configuration, so each can read what the others wrote. + +To use real S3, drop the endpoint override, credentials, and path-style setting from +`StorageConfig.java` and let the AWS SDK load its configuration from the environment instead. + +### Why a codec + +External storage works without a codec. Remove `ZlibPayloadCodec` and the order list is still +offloaded, only uncompressed. The sample uses one because many applications already run a codec, +and the two interact: + +- The threshold applies to the encoded payload. Compression alone can keep a payload small enough to + stay inline, so the threshold is a limit on the compressed size, not the size of your data. +- The driver stores the encoded payload. With an encryption codec in place of `ZlibPayloadCodec`, + the objects in S3 are encrypted too. +- The reference that replaces an offloaded payload in history is not passed through the codec, so + the bucket and key it names are visible to anyone who can read the history. diff --git a/core/src/main/java/io/temporal/samples/externalstorage/S3Mock.java b/core/src/main/java/io/temporal/samples/externalstorage/S3Mock.java new file mode 100644 index 000000000..c55afc23a --- /dev/null +++ b/core/src/main/java/io/temporal/samples/externalstorage/S3Mock.java @@ -0,0 +1,33 @@ +package io.temporal.samples.externalstorage; + +import java.net.URI; +import java.util.Properties; +import org.gaul.s3proxy.BlobStores; +import org.gaul.s3proxy.S3Proxy; +import org.gaul.s3proxy.auth.AuthenticationType; +import org.gaul.s3proxy.blobstore.BlobStore; + +/** A local S3 server that keeps objects in memory until it stops. */ +public class S3Mock { + + public static void main(String[] args) throws Exception { + BlobStore blobStore = BlobStores.create("transient", new Properties()); + blobStore.createContainer(StorageConfig.S3_BUCKET); + + S3Proxy proxy = + S3Proxy.builder() + .blobStore(blobStore) + .awsAuthentication( + AuthenticationType.AWS_V2_OR_V4, + StorageConfig.S3_ACCESS_KEY, + StorageConfig.S3_SECRET_KEY) + .endpoint(URI.create(StorageConfig.S3_ENDPOINT)) + .build(); + proxy.start(); + System.out.println( + "Mock S3 listening on " + + StorageConfig.S3_ENDPOINT + + ", bucket " + + StorageConfig.S3_BUCKET); + } +} diff --git a/core/src/main/java/io/temporal/samples/externalstorage/Starter.java b/core/src/main/java/io/temporal/samples/externalstorage/Starter.java new file mode 100644 index 000000000..d90e60ebc --- /dev/null +++ b/core/src/main/java/io/temporal/samples/externalstorage/Starter.java @@ -0,0 +1,42 @@ +package io.temporal.samples.externalstorage; + +import static io.temporal.samples.externalstorage.ExternalStorageWorker.TASK_QUEUE; + +import io.temporal.api.common.v1.WorkflowExecution; +import io.temporal.client.WorkflowClient; +import io.temporal.client.WorkflowOptions; +import io.temporal.client.WorkflowStub; +import io.temporal.envconfig.ClientConfigProfile; +import io.temporal.serviceclient.WorkflowServiceStubs; +import java.io.IOException; + +public class Starter { + + static final String WORKFLOW_ID = "external-storage-workflow-id"; + + public static void main(String[] args) throws IOException { + ClientConfigProfile profile = ClientConfigProfile.load(); + WorkflowServiceStubs service = + WorkflowServiceStubs.newServiceStubs(profile.toWorkflowServiceStubsOptions()); + WorkflowClient client = + WorkflowClient.newInstance( + service, StorageConfig.withExternalStorage(profile.toWorkflowClientOptions())); + + ProcessOrderBatchWorkflow workflow = + client.newWorkflowStub( + ProcessOrderBatchWorkflow.class, + WorkflowOptions.newBuilder() + .setWorkflowId(WORKFLOW_ID) + .setTaskQueue(TASK_QUEUE) + .build()); + + WorkflowExecution execution = WorkflowClient.start(workflow::run, 200); + System.out.println("Started workflow, run ID: " + execution.getRunId()); + + BatchSummary summary = WorkflowStub.fromTyped(workflow).getResult(BatchSummary.class); + System.out.printf( + "Processed %d orders weighing %.1f kg in total%n", + summary.orderCount(), summary.totalWeightKg()); + System.exit(0); + } +} diff --git a/core/src/main/java/io/temporal/samples/externalstorage/StorageConfig.java b/core/src/main/java/io/temporal/samples/externalstorage/StorageConfig.java new file mode 100644 index 000000000..06e52e8de --- /dev/null +++ b/core/src/main/java/io/temporal/samples/externalstorage/StorageConfig.java @@ -0,0 +1,57 @@ +package io.temporal.samples.externalstorage; + +import io.temporal.client.WorkflowClientOptions; +import io.temporal.common.converter.CodecDataConverter; +import io.temporal.common.converter.DefaultDataConverter; +import io.temporal.payload.codec.ZlibPayloadCodec; +import io.temporal.payload.storage.ExternalStorage; +import io.temporal.payload.storage.s3driver.S3StorageDriver; +import io.temporal.payload.storage.s3driver.awssdkv2.S3AsyncClientAdapter; +import java.net.URI; +import java.util.Collections; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; + +/** + * Every process in this sample must agree on the codec and storage configuration, so each can read + * what the others wrote. + */ +public final class StorageConfig { + + static final String S3_ENDPOINT = "http://127.0.0.1:5000"; + static final String S3_BUCKET = "temporal-payloads"; + static final String S3_ACCESS_KEY = "test"; + static final String S3_SECRET_KEY = "test"; + + private StorageConfig() {} + + /** Adds the zlib codec and S3 external storage to the given client options. */ + public static WorkflowClientOptions withExternalStorage(WorkflowClientOptions options) { + S3AsyncClient s3Client = + S3AsyncClient.builder() + .endpointOverride(URI.create(S3_ENDPOINT)) + .region(Region.US_EAST_1) + .credentialsProvider( + StaticCredentialsProvider.create( + AwsBasicCredentials.create(S3_ACCESS_KEY, S3_SECRET_KEY))) + // Virtual-hosted addressing would need temporal-payloads.127.0.0.1 to resolve. + .forcePathStyle(true) + .build(); + + S3StorageDriver driver = + S3StorageDriver.newBuilder() + .setClient(new S3AsyncClientAdapter(s3Client)) + .setBucket(S3_BUCKET) + .build(); + + return WorkflowClientOptions.newBuilder(options) + .setDataConverter( + new CodecDataConverter( + DefaultDataConverter.newDefaultInstance(), + Collections.singletonList(new ZlibPayloadCodec()))) + .setExternalStorage(ExternalStorage.newBuilder().setDriver(driver).build()) + .build(); + } +} diff --git a/core/src/main/resources/logback.xml b/core/src/main/resources/logback.xml index 418ee2447..3b4f761dc 100644 --- a/core/src/main/resources/logback.xml +++ b/core/src/main/resources/logback.xml @@ -28,6 +28,7 @@ + From 81728cd673c4d95b998732e7a33f3043ebcffee0 Mon Sep 17 00:00:00 2001 From: jmaeagle99 <44687433+jmaeagle99@users.noreply.github.com> Date: Tue, 10 Nov 2026 16:34:17 -0800 Subject: [PATCH 2/3] Add sample test --- .../samples/externalstorage/S3Mock.java | 21 ++++-- .../externalstorage/StorageConfig.java | 8 ++- .../externalstorage/ExternalStorageTest.java | 70 +++++++++++++++++++ 3 files changed, 91 insertions(+), 8 deletions(-) create mode 100644 core/src/test/java/io/temporal/samples/externalstorage/ExternalStorageTest.java diff --git a/core/src/main/java/io/temporal/samples/externalstorage/S3Mock.java b/core/src/main/java/io/temporal/samples/externalstorage/S3Mock.java index c55afc23a..3622f4420 100644 --- a/core/src/main/java/io/temporal/samples/externalstorage/S3Mock.java +++ b/core/src/main/java/io/temporal/samples/externalstorage/S3Mock.java @@ -11,6 +11,16 @@ public class S3Mock { public static void main(String[] args) throws Exception { + S3Proxy proxy = start(StorageConfig.S3_PORT); + System.out.println( + "Mock S3 listening on http://127.0.0.1:" + + proxy.getPort() + + ", bucket " + + StorageConfig.S3_BUCKET); + } + + /** Starts the server on the given port, or on a free one if the port is 0. */ + static S3Proxy start(int port) throws Exception { BlobStore blobStore = BlobStores.create("transient", new Properties()); blobStore.createContainer(StorageConfig.S3_BUCKET); @@ -21,13 +31,12 @@ public static void main(String[] args) throws Exception { AuthenticationType.AWS_V2_OR_V4, StorageConfig.S3_ACCESS_KEY, StorageConfig.S3_SECRET_KEY) - .endpoint(URI.create(StorageConfig.S3_ENDPOINT)) + .endpoint(URI.create("http://127.0.0.1:" + port)) .build(); proxy.start(); - System.out.println( - "Mock S3 listening on " - + StorageConfig.S3_ENDPOINT - + ", bucket " - + StorageConfig.S3_BUCKET); + while (!"STARTED".equals(proxy.getState())) { + Thread.sleep(10); + } + return proxy; } } diff --git a/core/src/main/java/io/temporal/samples/externalstorage/StorageConfig.java b/core/src/main/java/io/temporal/samples/externalstorage/StorageConfig.java index 06e52e8de..7ed6c48eb 100644 --- a/core/src/main/java/io/temporal/samples/externalstorage/StorageConfig.java +++ b/core/src/main/java/io/temporal/samples/externalstorage/StorageConfig.java @@ -20,7 +20,7 @@ */ public final class StorageConfig { - static final String S3_ENDPOINT = "http://127.0.0.1:5000"; + static final int S3_PORT = 5000; static final String S3_BUCKET = "temporal-payloads"; static final String S3_ACCESS_KEY = "test"; static final String S3_SECRET_KEY = "test"; @@ -29,9 +29,13 @@ private StorageConfig() {} /** Adds the zlib codec and S3 external storage to the given client options. */ public static WorkflowClientOptions withExternalStorage(WorkflowClientOptions options) { + return withExternalStorage(options, S3_PORT); + } + + static WorkflowClientOptions withExternalStorage(WorkflowClientOptions options, int s3Port) { S3AsyncClient s3Client = S3AsyncClient.builder() - .endpointOverride(URI.create(S3_ENDPOINT)) + .endpointOverride(URI.create("http://127.0.0.1:" + s3Port)) .region(Region.US_EAST_1) .credentialsProvider( StaticCredentialsProvider.create( diff --git a/core/src/test/java/io/temporal/samples/externalstorage/ExternalStorageTest.java b/core/src/test/java/io/temporal/samples/externalstorage/ExternalStorageTest.java new file mode 100644 index 000000000..8d7cbb0f1 --- /dev/null +++ b/core/src/test/java/io/temporal/samples/externalstorage/ExternalStorageTest.java @@ -0,0 +1,70 @@ +package io.temporal.samples.externalstorage; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import io.temporal.api.common.v1.Payload; +import io.temporal.api.common.v1.WorkflowExecution; +import io.temporal.api.history.v1.HistoryEvent; +import io.temporal.client.WorkflowClient; +import io.temporal.client.WorkflowClientOptions; +import io.temporal.client.WorkflowOptions; +import io.temporal.client.WorkflowStub; +import io.temporal.testing.TestWorkflowRule; +import org.gaul.s3proxy.S3Proxy; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; + +public class ExternalStorageTest { + + private static S3Proxy s3Mock; + + @BeforeClass + public static void startS3Mock() throws Exception { + s3Mock = S3Mock.start(0); + } + + @AfterClass + public static void stopS3Mock() throws Exception { + s3Mock.stop(); + } + + @Rule + public TestWorkflowRule testWorkflowRule = + TestWorkflowRule.newBuilder() + .setWorkflowTypes(ProcessOrderBatchWorkflowImpl.class) + .setActivityImplementations(new OrderActivitiesImpl()) + .setWorkflowClientOptions( + StorageConfig.withExternalStorage( + WorkflowClientOptions.newBuilder().build(), s3Mock.getPort())) + .build(); + + @Test + public void testOrderListIsOffloaded() { + ProcessOrderBatchWorkflow workflow = + testWorkflowRule + .getWorkflowClient() + .newWorkflowStub( + ProcessOrderBatchWorkflow.class, + WorkflowOptions.newBuilder().setTaskQueue(testWorkflowRule.getTaskQueue()).build()); + + WorkflowExecution execution = WorkflowClient.start(workflow::run, 200); + BatchSummary summary = WorkflowStub.fromTyped(workflow).getResult(BatchSummary.class); + assertEquals(200, summary.orderCount()); + assertEquals(5000.0, summary.totalWeightKg(), 0); + + // The sample only demonstrates anything if the order list is still above the threshold after + // compression. + Payload fetchOrdersResult = + testWorkflowRule.getHistory(execution).getEventsList().stream() + .filter(HistoryEvent::hasActivityTaskCompletedEventAttributes) + .findFirst() + .orElseThrow() + .getActivityTaskCompletedEventAttributes() + .getResult() + .getPayloads(0); + assertTrue(fetchOrdersResult.getExternalPayloadsCount() > 0); + } +} From bf862f170d470de7c569b433c959ef71075860ab Mon Sep 17 00:00:00 2001 From: jmaeagle99 <44687433+jmaeagle99@users.noreply.github.com> Date: Thu, 12 Nov 2026 10:29:26 -0800 Subject: [PATCH 3/3] Use real S3 via EXTERNAL_STORAGE_S3_BUCKET and move the mock to port 9100 --- .../samples/externalstorage/README.md | 22 +++++++-- .../samples/externalstorage/S3Mock.java | 10 ++-- .../externalstorage/StorageConfig.java | 46 ++++++++++++------- .../externalstorage/ExternalStorageTest.java | 4 +- 4 files changed, 54 insertions(+), 28 deletions(-) diff --git a/core/src/main/java/io/temporal/samples/externalstorage/README.md b/core/src/main/java/io/temporal/samples/externalstorage/README.md index 67eac9142..a299d90d4 100644 --- a/core/src/main/java/io/temporal/samples/externalstorage/README.md +++ b/core/src/main/java/io/temporal/samples/externalstorage/README.md @@ -15,12 +15,13 @@ The codec compresses each payload, and the SDK then compares the compressed size reference. The workflow's own input and result are a few bytes and stay inline. A mock S3 server ([S3Mock.java](S3Mock.java)), built on [S3Proxy](https://github.com/gaul/s3proxy), -is included, so the sample runs locally without an AWS account or Docker. +is included, so the sample runs locally without an AWS account or Docker. To use real S3 instead, +see [Using real S3](#using-real-s3). ## Running this sample 1. `temporal server start-dev` to start the Temporal server. -2. In another terminal, start the mock S3 server. It listens on `127.0.0.1:5000`, serves the +2. In another terminal, start the mock S3 server. It listens on `127.0.0.1:9100`, serves the `temporal-payloads` bucket, and keeps objects in memory until it stops. ```bash @@ -77,7 +78,7 @@ configuration, so it needs no other setup. S3StorageDriver driver = S3StorageDriver.newBuilder() .setClient(new S3AsyncClientAdapter(s3Client)) - .setBucket("temporal-payloads") + .setBucket(bucket) .build(); WorkflowClientOptions.newBuilder(options) @@ -103,8 +104,19 @@ Decoding reverses these steps, retrieving from S3 and then decompressing. Every process that reads or writes this workflow's payloads must use the same codec and external storage configuration, so each can read what the others wrote. -To use real S3, drop the endpoint override, credentials, and path-style setting from -`StorageConfig.java` and let the AWS SDK load its configuration from the environment instead. +### Using real S3 + +Set `EXTERNAL_STORAGE_S3_BUCKET` to your bucket's name in the worker's and the starter's terminals, +and skip the mock S3 server: + +```bash +export EXTERNAL_STORAGE_S3_BUCKET= +``` + +The S3 client then takes its region and credentials from the usual AWS configuration, such as the +`AWS_REGION` and `AWS_PROFILE` environment variables or `~/.aws`. Set the variable in both +terminals: a worker and a starter that disagree would write payloads where the other can't read +them. ### Why a codec diff --git a/core/src/main/java/io/temporal/samples/externalstorage/S3Mock.java b/core/src/main/java/io/temporal/samples/externalstorage/S3Mock.java index 3622f4420..e31baf369 100644 --- a/core/src/main/java/io/temporal/samples/externalstorage/S3Mock.java +++ b/core/src/main/java/io/temporal/samples/externalstorage/S3Mock.java @@ -11,26 +11,26 @@ public class S3Mock { public static void main(String[] args) throws Exception { - S3Proxy proxy = start(StorageConfig.S3_PORT); + S3Proxy proxy = start(StorageConfig.MOCK_PORT); System.out.println( "Mock S3 listening on http://127.0.0.1:" + proxy.getPort() + ", bucket " - + StorageConfig.S3_BUCKET); + + StorageConfig.MOCK_BUCKET); } /** Starts the server on the given port, or on a free one if the port is 0. */ static S3Proxy start(int port) throws Exception { BlobStore blobStore = BlobStores.create("transient", new Properties()); - blobStore.createContainer(StorageConfig.S3_BUCKET); + blobStore.createContainer(StorageConfig.MOCK_BUCKET); S3Proxy proxy = S3Proxy.builder() .blobStore(blobStore) .awsAuthentication( AuthenticationType.AWS_V2_OR_V4, - StorageConfig.S3_ACCESS_KEY, - StorageConfig.S3_SECRET_KEY) + StorageConfig.MOCK_ACCESS_KEY, + StorageConfig.MOCK_SECRET_KEY) .endpoint(URI.create("http://127.0.0.1:" + port)) .build(); proxy.start(); diff --git a/core/src/main/java/io/temporal/samples/externalstorage/StorageConfig.java b/core/src/main/java/io/temporal/samples/externalstorage/StorageConfig.java index 7ed6c48eb..c003bad46 100644 --- a/core/src/main/java/io/temporal/samples/externalstorage/StorageConfig.java +++ b/core/src/main/java/io/temporal/samples/externalstorage/StorageConfig.java @@ -20,34 +20,46 @@ */ public final class StorageConfig { - static final int S3_PORT = 5000; - static final String S3_BUCKET = "temporal-payloads"; - static final String S3_ACCESS_KEY = "test"; - static final String S3_SECRET_KEY = "test"; + /** Set to a bucket name to use real S3 instead of the local mock. */ + static final String BUCKET_ENV_VAR = "EXTERNAL_STORAGE_S3_BUCKET"; + + static final int MOCK_PORT = 9100; + static final String MOCK_BUCKET = "temporal-payloads"; + static final String MOCK_ACCESS_KEY = "test"; + static final String MOCK_SECRET_KEY = "test"; private StorageConfig() {} /** Adds the zlib codec and S3 external storage to the given client options. */ public static WorkflowClientOptions withExternalStorage(WorkflowClientOptions options) { - return withExternalStorage(options, S3_PORT); + String bucket = System.getenv(BUCKET_ENV_VAR); + if (bucket != null && !bucket.isEmpty()) { + // Region and credentials come from the usual AWS configuration, such as environment + // variables or ~/.aws. + return withExternalStorage(options, S3AsyncClient.create(), bucket); + } + return withExternalStorage(options, mockS3Client(MOCK_PORT), MOCK_BUCKET); } - static WorkflowClientOptions withExternalStorage(WorkflowClientOptions options, int s3Port) { - S3AsyncClient s3Client = - S3AsyncClient.builder() - .endpointOverride(URI.create("http://127.0.0.1:" + s3Port)) - .region(Region.US_EAST_1) - .credentialsProvider( - StaticCredentialsProvider.create( - AwsBasicCredentials.create(S3_ACCESS_KEY, S3_SECRET_KEY))) - // Virtual-hosted addressing would need temporal-payloads.127.0.0.1 to resolve. - .forcePathStyle(true) - .build(); + /** A client for the local mock S3 server started by {@link S3Mock}. */ + static S3AsyncClient mockS3Client(int port) { + return S3AsyncClient.builder() + .endpointOverride(URI.create("http://127.0.0.1:" + port)) + .region(Region.US_EAST_1) + .credentialsProvider( + StaticCredentialsProvider.create( + AwsBasicCredentials.create(MOCK_ACCESS_KEY, MOCK_SECRET_KEY))) + // Virtual-hosted addressing would need temporal-payloads.127.0.0.1 to resolve. + .forcePathStyle(true) + .build(); + } + static WorkflowClientOptions withExternalStorage( + WorkflowClientOptions options, S3AsyncClient s3Client, String bucket) { S3StorageDriver driver = S3StorageDriver.newBuilder() .setClient(new S3AsyncClientAdapter(s3Client)) - .setBucket(S3_BUCKET) + .setBucket(bucket) .build(); return WorkflowClientOptions.newBuilder(options) diff --git a/core/src/test/java/io/temporal/samples/externalstorage/ExternalStorageTest.java b/core/src/test/java/io/temporal/samples/externalstorage/ExternalStorageTest.java index 8d7cbb0f1..c3c2bc7d3 100644 --- a/core/src/test/java/io/temporal/samples/externalstorage/ExternalStorageTest.java +++ b/core/src/test/java/io/temporal/samples/externalstorage/ExternalStorageTest.java @@ -38,7 +38,9 @@ public static void stopS3Mock() throws Exception { .setActivityImplementations(new OrderActivitiesImpl()) .setWorkflowClientOptions( StorageConfig.withExternalStorage( - WorkflowClientOptions.newBuilder().build(), s3Mock.getPort())) + WorkflowClientOptions.newBuilder().build(), + StorageConfig.mockS3Client(s3Mock.getPort()), + StorageConfig.MOCK_BUCKET)) .build(); @Test