diff --git a/README.md b/README.md index b70a90983..2b4147987 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,8 @@ See the README.md file in each main sample directory for cut/paste Gradle comman - [**Worker Versioning**](/core/src/main/java/io/temporal/samples/workerversioning): Demonstrates how to use worker versioning to manage workflow code changes. +- [**External Storage**](/core/src/main/java/io/temporal/samples/externalstorage): Demonstrates offloading large payloads to S3 with a compression codec and a mock S3 server. + - [**Environment Configuration**](/core/src/main/java/io/temporal/samples/envconfig): Load client configuration from TOML files with programmatic overrides. diff --git a/core/build.gradle b/core/build.gradle index a06ff7f56..d2e6939e1 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -43,6 +43,11 @@ dependencies { implementation("software.amazon.awssdk:kms") implementation("software.amazon.awssdk:dynamodb") + // Used in External Storage sample + implementation "io.temporal:temporal-payload-storage-s3driver-awssdkv2:$javaSDKVersion" + implementation "software.amazon.awssdk:s3" + implementation "org.gaul:s3proxy:4.1.1" + // we don't update it to 2.1.0 because 2.1.0 requires Java 11 implementation 'com.codingrodent:jackson-json-crypto:1.1.0' diff --git a/core/src/main/java/io/temporal/samples/externalstorage/BatchSummary.java b/core/src/main/java/io/temporal/samples/externalstorage/BatchSummary.java new file mode 100644 index 000000000..49e54bd06 --- /dev/null +++ b/core/src/main/java/io/temporal/samples/externalstorage/BatchSummary.java @@ -0,0 +1,3 @@ +package io.temporal.samples.externalstorage; + +public record BatchSummary(int orderCount, double totalWeightKg) {} diff --git a/core/src/main/java/io/temporal/samples/externalstorage/ExternalStorageWorker.java b/core/src/main/java/io/temporal/samples/externalstorage/ExternalStorageWorker.java new file mode 100644 index 000000000..0300f137c --- /dev/null +++ b/core/src/main/java/io/temporal/samples/externalstorage/ExternalStorageWorker.java @@ -0,0 +1,32 @@ +package io.temporal.samples.externalstorage; + +import io.temporal.client.WorkflowClient; +import io.temporal.envconfig.ClientConfigProfile; +import io.temporal.serviceclient.WorkflowServiceStubs; +import io.temporal.worker.Worker; +import io.temporal.worker.WorkerFactory; +import java.io.IOException; + +public class ExternalStorageWorker { + + static final String TASK_QUEUE = "external-storage"; + + public static void main(String[] args) throws IOException { + ClientConfigProfile profile = ClientConfigProfile.load(); + WorkflowServiceStubs service = + WorkflowServiceStubs.newServiceStubs(profile.toWorkflowServiceStubsOptions()); + // The worker uses its client's data converter and external storage, so this is all it takes to + // offload the large activity inputs and results it sends. + WorkflowClient client = + WorkflowClient.newInstance( + service, StorageConfig.withExternalStorage(profile.toWorkflowClientOptions())); + + WorkerFactory factory = WorkerFactory.newInstance(client); + Worker worker = factory.newWorker(TASK_QUEUE); + worker.registerWorkflowImplementationTypes(ProcessOrderBatchWorkflowImpl.class); + worker.registerActivitiesImplementations(new OrderActivitiesImpl()); + + factory.start(); + System.out.println("Worker started on task queue: " + TASK_QUEUE); + } +} diff --git a/core/src/main/java/io/temporal/samples/externalstorage/Order.java b/core/src/main/java/io/temporal/samples/externalstorage/Order.java new file mode 100644 index 000000000..b0900cfb5 --- /dev/null +++ b/core/src/main/java/io/temporal/samples/externalstorage/Order.java @@ -0,0 +1,3 @@ +package io.temporal.samples.externalstorage; + +public record Order(String id, double weightKg, String notes) {} diff --git a/core/src/main/java/io/temporal/samples/externalstorage/OrderActivities.java b/core/src/main/java/io/temporal/samples/externalstorage/OrderActivities.java new file mode 100644 index 000000000..bfb486e06 --- /dev/null +++ b/core/src/main/java/io/temporal/samples/externalstorage/OrderActivities.java @@ -0,0 +1,12 @@ +package io.temporal.samples.externalstorage; + +import io.temporal.activity.ActivityInterface; +import java.util.List; + +@ActivityInterface +public interface OrderActivities { + + List fetchOrders(int orderCount); + + BatchSummary summarizeOrders(List orders); +} diff --git a/core/src/main/java/io/temporal/samples/externalstorage/OrderActivitiesImpl.java b/core/src/main/java/io/temporal/samples/externalstorage/OrderActivitiesImpl.java new file mode 100644 index 000000000..942e46786 --- /dev/null +++ b/core/src/main/java/io/temporal/samples/externalstorage/OrderActivitiesImpl.java @@ -0,0 +1,33 @@ +package io.temporal.samples.externalstorage; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +public class OrderActivitiesImpl implements OrderActivities { + + /** + * Random notes don't compress well, so the order list stays above the external storage threshold + * after the codec compresses it. + */ + private static final int NOTES_UUIDS_PER_ORDER = 100; + + @Override + public List fetchOrders(int orderCount) { + List orders = new ArrayList<>(orderCount); + for (int i = 1; i <= orderCount; i++) { + StringBuilder notes = new StringBuilder(); + for (int j = 0; j < NOTES_UUIDS_PER_ORDER; j++) { + notes.append(UUID.randomUUID()); + } + orders.add(new Order(String.format("ORD-%06d", i), i % 50 + 0.5, notes.toString())); + } + return orders; + } + + @Override + public BatchSummary summarizeOrders(List orders) { + double totalWeightKg = orders.stream().mapToDouble(Order::weightKg).sum(); + return new BatchSummary(orders.size(), totalWeightKg); + } +} diff --git a/core/src/main/java/io/temporal/samples/externalstorage/ProcessOrderBatchWorkflow.java b/core/src/main/java/io/temporal/samples/externalstorage/ProcessOrderBatchWorkflow.java new file mode 100644 index 000000000..142abb6ff --- /dev/null +++ b/core/src/main/java/io/temporal/samples/externalstorage/ProcessOrderBatchWorkflow.java @@ -0,0 +1,15 @@ +package io.temporal.samples.externalstorage; + +import io.temporal.workflow.WorkflowInterface; +import io.temporal.workflow.WorkflowMethod; + +/** + * The workflow's own input and result are small. The order list passed from {@code fetchOrders} to + * {@code summarizeOrders} is the payload large enough to be offloaded to external storage. + */ +@WorkflowInterface +public interface ProcessOrderBatchWorkflow { + + @WorkflowMethod + BatchSummary run(int orderCount); +} diff --git a/core/src/main/java/io/temporal/samples/externalstorage/ProcessOrderBatchWorkflowImpl.java b/core/src/main/java/io/temporal/samples/externalstorage/ProcessOrderBatchWorkflowImpl.java new file mode 100644 index 000000000..8bcbc3691 --- /dev/null +++ b/core/src/main/java/io/temporal/samples/externalstorage/ProcessOrderBatchWorkflowImpl.java @@ -0,0 +1,20 @@ +package io.temporal.samples.externalstorage; + +import io.temporal.activity.ActivityOptions; +import io.temporal.workflow.Workflow; +import java.time.Duration; +import java.util.List; + +public class ProcessOrderBatchWorkflowImpl implements ProcessOrderBatchWorkflow { + + private final OrderActivities activities = + Workflow.newActivityStub( + OrderActivities.class, + ActivityOptions.newBuilder().setStartToCloseTimeout(Duration.ofSeconds(60)).build()); + + @Override + public BatchSummary run(int orderCount) { + List orders = activities.fetchOrders(orderCount); + return activities.summarizeOrders(orders); + } +} diff --git a/core/src/main/java/io/temporal/samples/externalstorage/README.md b/core/src/main/java/io/temporal/samples/externalstorage/README.md new file mode 100644 index 000000000..a299d90d4 --- /dev/null +++ b/core/src/main/java/io/temporal/samples/externalstorage/README.md @@ -0,0 +1,132 @@ +# External Storage + +> **Note:** External storage is experimental and may change in the future. + +This sample offloads large payloads to S3-compatible object storage with the SDK's external storage +support and the `temporal-payload-storage-s3driver-awssdkv2` module. The SDK's `ZlibPayloadCodec` +compresses every payload first, so what is stored inline in Temporal and in S3 is compressed. + +**Scenario:** The workflow receives an order count and calls a `fetchOrders` activity that returns +that many orders. The list, about 400 KiB even after compression, is passed to a second +`summarizeOrders` activity, and the workflow returns its small `BatchSummary`. + +The codec compresses each payload, and the SDK then compares the compressed size against the default +256 KiB threshold. Payloads still above it are stored in S3 and replaced in history by a small +reference. The workflow's own input and result are a few bytes and stay inline. + +A mock S3 server ([S3Mock.java](S3Mock.java)), built on [S3Proxy](https://github.com/gaul/s3proxy), +is included, so the sample runs locally without an AWS account or Docker. To use real S3 instead, +see [Using real S3](#using-real-s3). + +## Running this sample + +1. `temporal server start-dev` to start the Temporal server. +2. In another terminal, start the mock S3 server. It listens on `127.0.0.1:9100`, serves the + `temporal-payloads` bucket, and keeps objects in memory until it stops. + + ```bash + ./gradlew -q :core:execute -PmainClass=io.temporal.samples.externalstorage.S3Mock + ``` + +3. In another terminal, start the worker: + + ```bash + ./gradlew -q :core:execute -PmainClass=io.temporal.samples.externalstorage.ExternalStorageWorker + ``` + +4. In another terminal, run the workflow: + + ```bash + ./gradlew -q :core:execute -PmainClass=io.temporal.samples.externalstorage.Starter + ``` + + Example output: + + ```text + Started workflow, run ID: + Processed 200 orders weighing 5000.0 kg in total + ``` + +5. Run `temporal workflow show` to see how the payloads were stored: + + ```bash + temporal workflow show --workflow-id external-storage-workflow-id + ``` + + | Payload | Type | Storage | + | ------------------------ | ------------------------ | ----------------- | + | Workflow input | `int` | Inline | + | `fetchOrders` input | `int` | Inline | + | `fetchOrders` result | `List` (~400 KiB) | **External (S3)** | + | `summarizeOrders` input | `List` (~400 KiB) | **External (S3)** | + | `summarizeOrders` result | `BatchSummary` | Inline | + | Workflow result | `BatchSummary` | Inline | + + The two offloaded payloads are the order list passed from one activity to the next, which is + what external storage is for: large data that flows between activities without bloating + workflow history. Both references name the same S3 object: the driver keys objects by a hash of + their content, so when the workflow passes the list on, it finds the object already stored and + skips the upload. + +## How it works + +[StorageConfig.java](StorageConfig.java) adds the codec and external storage to the +`WorkflowClientOptions` that both the worker and the starter use. The worker uses its client's +configuration, so it needs no other setup. + +```java +S3StorageDriver driver = + S3StorageDriver.newBuilder() + .setClient(new S3AsyncClientAdapter(s3Client)) + .setBucket(bucket) + .build(); + +WorkflowClientOptions.newBuilder(options) + .setDataConverter( + new CodecDataConverter( + DefaultDataConverter.newDefaultInstance(), + Collections.singletonList(new ZlibPayloadCodec()))) + .setExternalStorage(ExternalStorage.newBuilder().setDriver(driver).build()) + .build(); +``` + +When encoding a payload, the SDK: + +1. Serializes the value to a `Payload`. +2. Runs the codec, which compresses the payload. +3. Compares the compressed size against the threshold (default 256 KiB, set with + `ExternalStorage.Builder.setPayloadSizeThreshold`). +4. If it is at or above the threshold, stores the compressed payload in S3 and replaces it with a + reference. + +Decoding reverses these steps, retrieving from S3 and then decompressing. + +Every process that reads or writes this workflow's payloads must use the same codec and external +storage configuration, so each can read what the others wrote. + +### Using real S3 + +Set `EXTERNAL_STORAGE_S3_BUCKET` to your bucket's name in the worker's and the starter's terminals, +and skip the mock S3 server: + +```bash +export EXTERNAL_STORAGE_S3_BUCKET= +``` + +The S3 client then takes its region and credentials from the usual AWS configuration, such as the +`AWS_REGION` and `AWS_PROFILE` environment variables or `~/.aws`. Set the variable in both +terminals: a worker and a starter that disagree would write payloads where the other can't read +them. + +### Why a codec + +External storage works without a codec. Remove `ZlibPayloadCodec` and the order list is still +offloaded, only uncompressed. The sample uses one because many applications already run a codec, +and the two interact: + +- The threshold applies to the encoded payload. Compression alone can keep a payload small enough to + stay inline, so the threshold is a limit on the compressed size, not the size of your data. +- The driver stores the encoded payload. With an encryption codec in place of `ZlibPayloadCodec`, + the objects in S3 are encrypted too. +- The reference that replaces an offloaded payload in history is not passed through the codec, so + the bucket and key it names are visible to anyone who can read the history. diff --git a/core/src/main/java/io/temporal/samples/externalstorage/S3Mock.java b/core/src/main/java/io/temporal/samples/externalstorage/S3Mock.java new file mode 100644 index 000000000..e31baf369 --- /dev/null +++ b/core/src/main/java/io/temporal/samples/externalstorage/S3Mock.java @@ -0,0 +1,42 @@ +package io.temporal.samples.externalstorage; + +import java.net.URI; +import java.util.Properties; +import org.gaul.s3proxy.BlobStores; +import org.gaul.s3proxy.S3Proxy; +import org.gaul.s3proxy.auth.AuthenticationType; +import org.gaul.s3proxy.blobstore.BlobStore; + +/** A local S3 server that keeps objects in memory until it stops. */ +public class S3Mock { + + public static void main(String[] args) throws Exception { + S3Proxy proxy = start(StorageConfig.MOCK_PORT); + System.out.println( + "Mock S3 listening on http://127.0.0.1:" + + proxy.getPort() + + ", bucket " + + StorageConfig.MOCK_BUCKET); + } + + /** Starts the server on the given port, or on a free one if the port is 0. */ + static S3Proxy start(int port) throws Exception { + BlobStore blobStore = BlobStores.create("transient", new Properties()); + blobStore.createContainer(StorageConfig.MOCK_BUCKET); + + S3Proxy proxy = + S3Proxy.builder() + .blobStore(blobStore) + .awsAuthentication( + AuthenticationType.AWS_V2_OR_V4, + StorageConfig.MOCK_ACCESS_KEY, + StorageConfig.MOCK_SECRET_KEY) + .endpoint(URI.create("http://127.0.0.1:" + port)) + .build(); + proxy.start(); + while (!"STARTED".equals(proxy.getState())) { + Thread.sleep(10); + } + return proxy; + } +} diff --git a/core/src/main/java/io/temporal/samples/externalstorage/Starter.java b/core/src/main/java/io/temporal/samples/externalstorage/Starter.java new file mode 100644 index 000000000..d90e60ebc --- /dev/null +++ b/core/src/main/java/io/temporal/samples/externalstorage/Starter.java @@ -0,0 +1,42 @@ +package io.temporal.samples.externalstorage; + +import static io.temporal.samples.externalstorage.ExternalStorageWorker.TASK_QUEUE; + +import io.temporal.api.common.v1.WorkflowExecution; +import io.temporal.client.WorkflowClient; +import io.temporal.client.WorkflowOptions; +import io.temporal.client.WorkflowStub; +import io.temporal.envconfig.ClientConfigProfile; +import io.temporal.serviceclient.WorkflowServiceStubs; +import java.io.IOException; + +public class Starter { + + static final String WORKFLOW_ID = "external-storage-workflow-id"; + + public static void main(String[] args) throws IOException { + ClientConfigProfile profile = ClientConfigProfile.load(); + WorkflowServiceStubs service = + WorkflowServiceStubs.newServiceStubs(profile.toWorkflowServiceStubsOptions()); + WorkflowClient client = + WorkflowClient.newInstance( + service, StorageConfig.withExternalStorage(profile.toWorkflowClientOptions())); + + ProcessOrderBatchWorkflow workflow = + client.newWorkflowStub( + ProcessOrderBatchWorkflow.class, + WorkflowOptions.newBuilder() + .setWorkflowId(WORKFLOW_ID) + .setTaskQueue(TASK_QUEUE) + .build()); + + WorkflowExecution execution = WorkflowClient.start(workflow::run, 200); + System.out.println("Started workflow, run ID: " + execution.getRunId()); + + BatchSummary summary = WorkflowStub.fromTyped(workflow).getResult(BatchSummary.class); + System.out.printf( + "Processed %d orders weighing %.1f kg in total%n", + summary.orderCount(), summary.totalWeightKg()); + System.exit(0); + } +} diff --git a/core/src/main/java/io/temporal/samples/externalstorage/StorageConfig.java b/core/src/main/java/io/temporal/samples/externalstorage/StorageConfig.java new file mode 100644 index 000000000..c003bad46 --- /dev/null +++ b/core/src/main/java/io/temporal/samples/externalstorage/StorageConfig.java @@ -0,0 +1,73 @@ +package io.temporal.samples.externalstorage; + +import io.temporal.client.WorkflowClientOptions; +import io.temporal.common.converter.CodecDataConverter; +import io.temporal.common.converter.DefaultDataConverter; +import io.temporal.payload.codec.ZlibPayloadCodec; +import io.temporal.payload.storage.ExternalStorage; +import io.temporal.payload.storage.s3driver.S3StorageDriver; +import io.temporal.payload.storage.s3driver.awssdkv2.S3AsyncClientAdapter; +import java.net.URI; +import java.util.Collections; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; + +/** + * Every process in this sample must agree on the codec and storage configuration, so each can read + * what the others wrote. + */ +public final class StorageConfig { + + /** Set to a bucket name to use real S3 instead of the local mock. */ + static final String BUCKET_ENV_VAR = "EXTERNAL_STORAGE_S3_BUCKET"; + + static final int MOCK_PORT = 9100; + static final String MOCK_BUCKET = "temporal-payloads"; + static final String MOCK_ACCESS_KEY = "test"; + static final String MOCK_SECRET_KEY = "test"; + + private StorageConfig() {} + + /** Adds the zlib codec and S3 external storage to the given client options. */ + public static WorkflowClientOptions withExternalStorage(WorkflowClientOptions options) { + String bucket = System.getenv(BUCKET_ENV_VAR); + if (bucket != null && !bucket.isEmpty()) { + // Region and credentials come from the usual AWS configuration, such as environment + // variables or ~/.aws. + return withExternalStorage(options, S3AsyncClient.create(), bucket); + } + return withExternalStorage(options, mockS3Client(MOCK_PORT), MOCK_BUCKET); + } + + /** A client for the local mock S3 server started by {@link S3Mock}. */ + static S3AsyncClient mockS3Client(int port) { + return S3AsyncClient.builder() + .endpointOverride(URI.create("http://127.0.0.1:" + port)) + .region(Region.US_EAST_1) + .credentialsProvider( + StaticCredentialsProvider.create( + AwsBasicCredentials.create(MOCK_ACCESS_KEY, MOCK_SECRET_KEY))) + // Virtual-hosted addressing would need temporal-payloads.127.0.0.1 to resolve. + .forcePathStyle(true) + .build(); + } + + static WorkflowClientOptions withExternalStorage( + WorkflowClientOptions options, S3AsyncClient s3Client, String bucket) { + S3StorageDriver driver = + S3StorageDriver.newBuilder() + .setClient(new S3AsyncClientAdapter(s3Client)) + .setBucket(bucket) + .build(); + + return WorkflowClientOptions.newBuilder(options) + .setDataConverter( + new CodecDataConverter( + DefaultDataConverter.newDefaultInstance(), + Collections.singletonList(new ZlibPayloadCodec()))) + .setExternalStorage(ExternalStorage.newBuilder().setDriver(driver).build()) + .build(); + } +} diff --git a/core/src/main/resources/logback.xml b/core/src/main/resources/logback.xml index 418ee2447..3b4f761dc 100644 --- a/core/src/main/resources/logback.xml +++ b/core/src/main/resources/logback.xml @@ -28,6 +28,7 @@ + diff --git a/core/src/test/java/io/temporal/samples/externalstorage/ExternalStorageTest.java b/core/src/test/java/io/temporal/samples/externalstorage/ExternalStorageTest.java new file mode 100644 index 000000000..c3c2bc7d3 --- /dev/null +++ b/core/src/test/java/io/temporal/samples/externalstorage/ExternalStorageTest.java @@ -0,0 +1,72 @@ +package io.temporal.samples.externalstorage; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import io.temporal.api.common.v1.Payload; +import io.temporal.api.common.v1.WorkflowExecution; +import io.temporal.api.history.v1.HistoryEvent; +import io.temporal.client.WorkflowClient; +import io.temporal.client.WorkflowClientOptions; +import io.temporal.client.WorkflowOptions; +import io.temporal.client.WorkflowStub; +import io.temporal.testing.TestWorkflowRule; +import org.gaul.s3proxy.S3Proxy; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; + +public class ExternalStorageTest { + + private static S3Proxy s3Mock; + + @BeforeClass + public static void startS3Mock() throws Exception { + s3Mock = S3Mock.start(0); + } + + @AfterClass + public static void stopS3Mock() throws Exception { + s3Mock.stop(); + } + + @Rule + public TestWorkflowRule testWorkflowRule = + TestWorkflowRule.newBuilder() + .setWorkflowTypes(ProcessOrderBatchWorkflowImpl.class) + .setActivityImplementations(new OrderActivitiesImpl()) + .setWorkflowClientOptions( + StorageConfig.withExternalStorage( + WorkflowClientOptions.newBuilder().build(), + StorageConfig.mockS3Client(s3Mock.getPort()), + StorageConfig.MOCK_BUCKET)) + .build(); + + @Test + public void testOrderListIsOffloaded() { + ProcessOrderBatchWorkflow workflow = + testWorkflowRule + .getWorkflowClient() + .newWorkflowStub( + ProcessOrderBatchWorkflow.class, + WorkflowOptions.newBuilder().setTaskQueue(testWorkflowRule.getTaskQueue()).build()); + + WorkflowExecution execution = WorkflowClient.start(workflow::run, 200); + BatchSummary summary = WorkflowStub.fromTyped(workflow).getResult(BatchSummary.class); + assertEquals(200, summary.orderCount()); + assertEquals(5000.0, summary.totalWeightKg(), 0); + + // The sample only demonstrates anything if the order list is still above the threshold after + // compression. + Payload fetchOrdersResult = + testWorkflowRule.getHistory(execution).getEventsList().stream() + .filter(HistoryEvent::hasActivityTaskCompletedEventAttributes) + .findFirst() + .orElseThrow() + .getActivityTaskCompletedEventAttributes() + .getResult() + .getPayloads(0); + assertTrue(fetchOrdersResult.getExternalPayloadsCount() > 0); + } +}