Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,8 @@ See the README.md file in each main sample directory for cut/paste Gradle comman

- [**Worker Versioning**](/core/src/main/java/io/temporal/samples/workerversioning): Demonstrates how to use worker versioning to manage workflow code changes.

- [**External Storage**](/core/src/main/java/io/temporal/samples/externalstorage): Demonstrates offloading large payloads to S3 with a compression codec and a mock S3 server.

- [**Environment Configuration**](/core/src/main/java/io/temporal/samples/envconfig):
Load client configuration from TOML files with programmatic overrides.

Expand Down
5 changes: 5 additions & 0 deletions core/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,11 @@ dependencies {
implementation("software.amazon.awssdk:kms")
implementation("software.amazon.awssdk:dynamodb")

// Used in External Storage sample
implementation "io.temporal:temporal-payload-storage-s3driver-awssdkv2:$javaSDKVersion"
implementation "software.amazon.awssdk:s3"
implementation "org.gaul:s3proxy:4.1.1"

// we don't update it to 2.1.0 because 2.1.0 requires Java 11
implementation 'com.codingrodent:jackson-json-crypto:1.1.0'

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
package io.temporal.samples.externalstorage;

public record BatchSummary(int orderCount, double totalWeightKg) {}
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
package io.temporal.samples.externalstorage;

import io.temporal.client.WorkflowClient;
import io.temporal.envconfig.ClientConfigProfile;
import io.temporal.serviceclient.WorkflowServiceStubs;
import io.temporal.worker.Worker;
import io.temporal.worker.WorkerFactory;
import java.io.IOException;

public class ExternalStorageWorker {

static final String TASK_QUEUE = "external-storage";

public static void main(String[] args) throws IOException {
ClientConfigProfile profile = ClientConfigProfile.load();
WorkflowServiceStubs service =
WorkflowServiceStubs.newServiceStubs(profile.toWorkflowServiceStubsOptions());
// The worker uses its client's data converter and external storage, so this is all it takes to
// offload the large activity inputs and results it sends.
WorkflowClient client =
WorkflowClient.newInstance(
service, StorageConfig.withExternalStorage(profile.toWorkflowClientOptions()));

WorkerFactory factory = WorkerFactory.newInstance(client);
Worker worker = factory.newWorker(TASK_QUEUE);
worker.registerWorkflowImplementationTypes(ProcessOrderBatchWorkflowImpl.class);
worker.registerActivitiesImplementations(new OrderActivitiesImpl());

factory.start();
System.out.println("Worker started on task queue: " + TASK_QUEUE);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
package io.temporal.samples.externalstorage;

public record Order(String id, double weightKg, String notes) {}
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
package io.temporal.samples.externalstorage;

import io.temporal.activity.ActivityInterface;
import java.util.List;

@ActivityInterface
public interface OrderActivities {

List<Order> fetchOrders(int orderCount);

BatchSummary summarizeOrders(List<Order> orders);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package io.temporal.samples.externalstorage;

import java.util.ArrayList;
import java.util.List;
import java.util.UUID;

public class OrderActivitiesImpl implements OrderActivities {

/**
* Random notes don't compress well, so the order list stays above the external storage threshold
* after the codec compresses it.
*/
private static final int NOTES_UUIDS_PER_ORDER = 100;

@Override
public List<Order> fetchOrders(int orderCount) {
List<Order> orders = new ArrayList<>(orderCount);
for (int i = 1; i <= orderCount; i++) {
StringBuilder notes = new StringBuilder();
for (int j = 0; j < NOTES_UUIDS_PER_ORDER; j++) {
notes.append(UUID.randomUUID());
}
orders.add(new Order(String.format("ORD-%06d", i), i % 50 + 0.5, notes.toString()));
}
return orders;
}

@Override
public BatchSummary summarizeOrders(List<Order> orders) {
double totalWeightKg = orders.stream().mapToDouble(Order::weightKg).sum();
return new BatchSummary(orders.size(), totalWeightKg);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package io.temporal.samples.externalstorage;

import io.temporal.workflow.WorkflowInterface;
import io.temporal.workflow.WorkflowMethod;

/**
* The workflow's own input and result are small. The order list passed from {@code fetchOrders} to
* {@code summarizeOrders} is the payload large enough to be offloaded to external storage.
*/
@WorkflowInterface
public interface ProcessOrderBatchWorkflow {

@WorkflowMethod
BatchSummary run(int orderCount);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
package io.temporal.samples.externalstorage;

import io.temporal.activity.ActivityOptions;
import io.temporal.workflow.Workflow;
import java.time.Duration;
import java.util.List;

public class ProcessOrderBatchWorkflowImpl implements ProcessOrderBatchWorkflow {

private final OrderActivities activities =
Workflow.newActivityStub(
OrderActivities.class,
ActivityOptions.newBuilder().setStartToCloseTimeout(Duration.ofSeconds(60)).build());

@Override
public BatchSummary run(int orderCount) {
List<Order> orders = activities.fetchOrders(orderCount);
return activities.summarizeOrders(orders);
}
}
132 changes: 132 additions & 0 deletions core/src/main/java/io/temporal/samples/externalstorage/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
# External Storage

> **Note:** External storage is experimental and may change in the future.

This sample offloads large payloads to S3-compatible object storage with the SDK's external storage
support and the `temporal-payload-storage-s3driver-awssdkv2` module. The SDK's `ZlibPayloadCodec`
compresses every payload first, so what is stored inline in Temporal and in S3 is compressed.

**Scenario:** The workflow receives an order count and calls a `fetchOrders` activity that returns
that many orders. The list, about 400 KiB even after compression, is passed to a second
`summarizeOrders` activity, and the workflow returns its small `BatchSummary`.

The codec compresses each payload, and the SDK then compares the compressed size against the default
256 KiB threshold. Payloads still above it are stored in S3 and replaced in history by a small
reference. The workflow's own input and result are a few bytes and stay inline.

A mock S3 server ([S3Mock.java](S3Mock.java)), built on [S3Proxy](https://github.com/gaul/s3proxy),
is included, so the sample runs locally without an AWS account or Docker. To use real S3 instead,
see [Using real S3](#using-real-s3).

## Running this sample

1. `temporal server start-dev` to start the Temporal server.
2. In another terminal, start the mock S3 server. It listens on `127.0.0.1:9100`, serves the
`temporal-payloads` bucket, and keeps objects in memory until it stops.

```bash
./gradlew -q :core:execute -PmainClass=io.temporal.samples.externalstorage.S3Mock
```

3. In another terminal, start the worker:

```bash
./gradlew -q :core:execute -PmainClass=io.temporal.samples.externalstorage.ExternalStorageWorker
```

4. In another terminal, run the workflow:

```bash
./gradlew -q :core:execute -PmainClass=io.temporal.samples.externalstorage.Starter
```

Example output:

```text
Started workflow, run ID: <run-id>
Processed 200 orders weighing 5000.0 kg in total
```

5. Run `temporal workflow show` to see how the payloads were stored:

```bash
temporal workflow show --workflow-id external-storage-workflow-id
```

| Payload | Type | Storage |
| ------------------------ | ------------------------ | ----------------- |
| Workflow input | `int` | Inline |
| `fetchOrders` input | `int` | Inline |
| `fetchOrders` result | `List<Order>` (~400 KiB) | **External (S3)** |
| `summarizeOrders` input | `List<Order>` (~400 KiB) | **External (S3)** |
| `summarizeOrders` result | `BatchSummary` | Inline |
| Workflow result | `BatchSummary` | Inline |

The two offloaded payloads are the order list passed from one activity to the next, which is
what external storage is for: large data that flows between activities without bloating
workflow history. Both references name the same S3 object: the driver keys objects by a hash of
their content, so when the workflow passes the list on, it finds the object already stored and
skips the upload.

## How it works

[StorageConfig.java](StorageConfig.java) adds the codec and external storage to the
`WorkflowClientOptions` that both the worker and the starter use. The worker uses its client's
configuration, so it needs no other setup.

```java
S3StorageDriver driver =
S3StorageDriver.newBuilder()
.setClient(new S3AsyncClientAdapter(s3Client))
.setBucket(bucket)
.build();

WorkflowClientOptions.newBuilder(options)
.setDataConverter(
new CodecDataConverter(
DefaultDataConverter.newDefaultInstance(),
Collections.singletonList(new ZlibPayloadCodec())))
.setExternalStorage(ExternalStorage.newBuilder().setDriver(driver).build())
.build();
```

When encoding a payload, the SDK:

1. Serializes the value to a `Payload`.
2. Runs the codec, which compresses the payload.
3. Compares the compressed size against the threshold (default 256 KiB, set with
`ExternalStorage.Builder.setPayloadSizeThreshold`).
4. If it is at or above the threshold, stores the compressed payload in S3 and replaces it with a
reference.

Decoding reverses these steps, retrieving from S3 and then decompressing.

Every process that reads or writes this workflow's payloads must use the same codec and external
storage configuration, so each can read what the others wrote.

### Using real S3

Set `EXTERNAL_STORAGE_S3_BUCKET` to your bucket's name in the worker's and the starter's terminals,
and skip the mock S3 server:

```bash
export EXTERNAL_STORAGE_S3_BUCKET=<your-bucket>
```

The S3 client then takes its region and credentials from the usual AWS configuration, such as the
`AWS_REGION` and `AWS_PROFILE` environment variables or `~/.aws`. Set the variable in both
terminals: a worker and a starter that disagree would write payloads where the other can't read
them.

### Why a codec

External storage works without a codec. Remove `ZlibPayloadCodec` and the order list is still
offloaded, only uncompressed. The sample uses one because many applications already run a codec,
and the two interact:

- The threshold applies to the encoded payload. Compression alone can keep a payload small enough to
stay inline, so the threshold is a limit on the compressed size, not the size of your data.
- The driver stores the encoded payload. With an encryption codec in place of `ZlibPayloadCodec`,
the objects in S3 are encrypted too.
- The reference that replaces an offloaded payload in history is not passed through the codec, so
the bucket and key it names are visible to anyone who can read the history.
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
package io.temporal.samples.externalstorage;

import java.net.URI;
import java.util.Properties;
import org.gaul.s3proxy.BlobStores;
import org.gaul.s3proxy.S3Proxy;
import org.gaul.s3proxy.auth.AuthenticationType;
import org.gaul.s3proxy.blobstore.BlobStore;

/** A local S3 server that keeps objects in memory until it stops. */
public class S3Mock {

public static void main(String[] args) throws Exception {
S3Proxy proxy = start(StorageConfig.MOCK_PORT);
System.out.println(
"Mock S3 listening on http://127.0.0.1:"
+ proxy.getPort()
+ ", bucket "
+ StorageConfig.MOCK_BUCKET);
}

/** Starts the server on the given port, or on a free one if the port is 0. */
static S3Proxy start(int port) throws Exception {
BlobStore blobStore = BlobStores.create("transient", new Properties());
blobStore.createContainer(StorageConfig.MOCK_BUCKET);

S3Proxy proxy =
S3Proxy.builder()
.blobStore(blobStore)
.awsAuthentication(
AuthenticationType.AWS_V2_OR_V4,
StorageConfig.MOCK_ACCESS_KEY,
StorageConfig.MOCK_SECRET_KEY)
.endpoint(URI.create("http://127.0.0.1:" + port))
.build();
proxy.start();
while (!"STARTED".equals(proxy.getState())) {
Thread.sleep(10);
}
return proxy;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
package io.temporal.samples.externalstorage;

import static io.temporal.samples.externalstorage.ExternalStorageWorker.TASK_QUEUE;

import io.temporal.api.common.v1.WorkflowExecution;
import io.temporal.client.WorkflowClient;
import io.temporal.client.WorkflowOptions;
import io.temporal.client.WorkflowStub;
import io.temporal.envconfig.ClientConfigProfile;
import io.temporal.serviceclient.WorkflowServiceStubs;
import java.io.IOException;

public class Starter {

static final String WORKFLOW_ID = "external-storage-workflow-id";

public static void main(String[] args) throws IOException {
ClientConfigProfile profile = ClientConfigProfile.load();
WorkflowServiceStubs service =
WorkflowServiceStubs.newServiceStubs(profile.toWorkflowServiceStubsOptions());
WorkflowClient client =
WorkflowClient.newInstance(
service, StorageConfig.withExternalStorage(profile.toWorkflowClientOptions()));

ProcessOrderBatchWorkflow workflow =
client.newWorkflowStub(
ProcessOrderBatchWorkflow.class,
WorkflowOptions.newBuilder()
.setWorkflowId(WORKFLOW_ID)
.setTaskQueue(TASK_QUEUE)
.build());

WorkflowExecution execution = WorkflowClient.start(workflow::run, 200);
System.out.println("Started workflow, run ID: " + execution.getRunId());

BatchSummary summary = WorkflowStub.fromTyped(workflow).getResult(BatchSummary.class);
System.out.printf(
"Processed %d orders weighing %.1f kg in total%n",
summary.orderCount(), summary.totalWeightKg());
System.exit(0);
}
}
Loading
Loading