From fbf07ba806515ae6858bd273ba81a0049b75334c Mon Sep 17 00:00:00 2001 From: johanngan Date: Sat, 14 Nov 2026 22:38:28 -0600 Subject: [PATCH 1/2] Add AWS Secrets Manager envconfig extension Add the experimental temporal-aws-secrets-manager module, an envconfig extension that loads the client API key and mTLS material from AWS Secrets Manager secret IDs, configured in TOML or with TEMPORAL_EXT_AWS_SECRETS_MANAGER_* environment variables. It ports the Go, Python and TypeScript implementations, with the same config, validation, metric names and log messages. Secrets are fetched through the AWS Parameters and Secrets Lambda Extension when it's detected, and otherwise with the AWS SDK for Java 2.x. The AWS SDK is a compile-only dependency that users add when they need it, and only SdkFetcher references it, so the module loads without it. Fetched secrets are cached, refreshed every refresh interval (SECRETS_MANAGER_TTL, default 5 minutes), and fall back to the previous value for up to 24 hours if a refresh fails. The plugin fetches the secrets when service stubs are created. The API key supplier returns the cached key and refreshes it in the background, so requests never wait on a fetch. TLS secrets are combined with the profile's other TLS settings into a new SSL context, since an SslContext can't be extended. Client keys must be PKCS#8, as for envconfig, and a client cert and key that don't match are treated as a failed fetch. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 + build.gradle | 3 + .../temporal-aws-secrets-manager/README.md | 149 ++++ .../temporal-aws-secrets-manager/build.gradle | 23 + .../aws/secretsmanager/FetchErrors.java | 47 ++ .../temporal/aws/secretsmanager/Fetcher.java | 116 ++++ .../LambdaExtensionFetcher.java | 129 ++++ .../aws/secretsmanager/SdkFetcher.java | 76 +++ .../aws/secretsmanager/SecretCache.java | 646 ++++++++++++++++++ .../SecretsManagerExtension.java | 352 ++++++++++ .../secretsmanager/SecretsManagerPlugin.java | 163 +++++ .../aws/secretsmanager/Telemetry.java | 109 +++ .../aws/secretsmanager/FetcherTest.java | 249 +++++++ .../aws/secretsmanager/SecretCacheTest.java | 396 +++++++++++ .../SecretsManagerExtensionTest.java | 499 ++++++++++++++ .../aws/secretsmanager/TestHelpers.java | 183 +++++ .../build.gradle | 4 - settings.gradle | 2 + temporal-bom/build.gradle | 1 + 19 files changed, 3145 insertions(+), 4 deletions(-) create mode 100644 contrib/temporal-aws-secrets-manager/README.md create mode 100644 contrib/temporal-aws-secrets-manager/build.gradle create mode 100644 contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/FetchErrors.java create mode 100644 contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/Fetcher.java create mode 100644 contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/LambdaExtensionFetcher.java create mode 100644 contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/SdkFetcher.java create mode 100644 contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/SecretCache.java create mode 100644 contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/SecretsManagerExtension.java create mode 100644 contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/SecretsManagerPlugin.java create mode 100644 contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/Telemetry.java create mode 100644 contrib/temporal-aws-secrets-manager/src/test/java/io/temporal/aws/secretsmanager/FetcherTest.java create mode 100644 contrib/temporal-aws-secrets-manager/src/test/java/io/temporal/aws/secretsmanager/SecretCacheTest.java create mode 100644 contrib/temporal-aws-secrets-manager/src/test/java/io/temporal/aws/secretsmanager/SecretsManagerExtensionTest.java create mode 100644 contrib/temporal-aws-secrets-manager/src/test/java/io/temporal/aws/secretsmanager/TestHelpers.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 9153295fe1..5f91e4c14d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,8 @@ to docs, or any other relevant information. - Experimental Event Groups associate related workflow commands so their history events can be viewed together. Create a group with `Workflow.newEventGroup` and attach it by calling `addEventGroups` on the command's options. +- Added the experimental `temporal-aws-secrets-manager` module, a `temporal-envconfig` extension that loads the client + API key and mTLS material from AWS Secrets Manager secret IDs. ### Changed - Updated Cloud API bindings to v0.27.0, including opt-in user group memberships, projects, diff --git a/build.gradle b/build.gradle index feb580323b..f0b07a2d64 100644 --- a/build.gradle +++ b/build.gradle @@ -46,6 +46,9 @@ ext { jsonPathVersion = '2.9.0' // compileOnly + // api in temporal-payload-storage-s3driver-awssdkv2, compileOnly in temporal-aws-secrets-manager + awsSdkVersion = '2.31.0' + cronUtilsVersion = '9.2.1' // for test server only // This is the version of Spring Boot we use to _build the SDK_. This has to be in the diff --git a/contrib/temporal-aws-secrets-manager/README.md b/contrib/temporal-aws-secrets-manager/README.md new file mode 100644 index 0000000000..1bc4744e81 --- /dev/null +++ b/contrib/temporal-aws-secrets-manager/README.md @@ -0,0 +1,149 @@ +# AWS Secrets Manager for Temporal Client Config + +> ⚠️ **This module is currently at an experimental release stage.** ⚠️ + +`temporal-aws-secrets-manager` is a [`temporal-envconfig`](../../temporal-envconfig) extension that loads Temporal +client credentials (API key and/or mTLS material) from [AWS Secrets Manager](https://aws.amazon.com/secrets-manager/). +Only secret IDs live in your config. Secret material is fetched at runtime and kept in memory. + +## Install + +Add `io.temporal:temporal-aws-secrets-manager` next to your Temporal SDK dependency. + +If the +[AWS Parameters and Secrets Lambda Extension](https://docs.aws.amazon.com/secretsmanager/latest/userguide/retrieving-secrets_lambda.html) +is available (see [Fetching and caching](#fetching-and-caching)), no other dependencies are needed. Otherwise, secrets +are fetched directly with the AWS SDK for Java 2.x, which is an optional dependency that you add yourself: + +```groovy +implementation "software.amazon.awssdk:secretsmanager:" +``` + +If it's needed and missing, loading the config fails with an error saying so. + +## Usage + +```java +ClientConfigProfile profile = + ClientConfigProfile.load( + LoadClientConfigProfileOptions.newBuilder() + .setExtensions(Collections.singletonList(SecretsManagerExtension.getDefaultInstance())) + .build()); +WorkflowServiceStubs service = + WorkflowServiceStubs.newServiceStubs(profile.toWorkflowServiceStubsOptions()); +WorkflowClient client = WorkflowClient.newInstance(service, profile.toWorkflowClientOptions()); +``` + +The extension adds a plugin to the profile's plugins, which are set on the options returned by +`toWorkflowServiceStubsOptions()`. The plugin fetches the secrets when service stubs are created. To use other service +stubs plugins, add them to these rather than replacing them. + +## Configuration + +Each credential is configured with a Secrets Manager secret ID (friendly name or full ARN), either in the TOML config +file or with environment variables. Environment variables take precedence over the file. + +| TOML (under `profile..ext.aws_secrets_manager`) | Environment variable | Replaces | +|---|---|---| +| `api_key_id` | `TEMPORAL_EXT_AWS_SECRETS_MANAGER_API_KEY_ID` | `api_key` / `TEMPORAL_API_KEY` | +| `tls.client_cert_id` | `TEMPORAL_EXT_AWS_SECRETS_MANAGER_TLS_CLIENT_CERT_ID` | `tls.client_cert_{path,data}` / `TEMPORAL_TLS_CLIENT_CERT_{PATH,DATA}` | +| `tls.client_key_id` | `TEMPORAL_EXT_AWS_SECRETS_MANAGER_TLS_CLIENT_KEY_ID` | `tls.client_key_{path,data}` / `TEMPORAL_TLS_CLIENT_KEY_{PATH,DATA}` | +| `tls.server_ca_cert_id` | `TEMPORAL_EXT_AWS_SECRETS_MANAGER_TLS_SERVER_CA_CERT_ID` | `tls.server_ca_cert_{path,data}` / `TEMPORAL_TLS_SERVER_CA_CERT_{PATH,DATA}` | + +For example: + +```toml +[profile.default] +address = "my-namespace.a1b2c.tmprl.cloud:7233" +namespace = "my-namespace.a1b2c" + +[profile.default.ext.aws_secrets_manager.tls] +client_cert_id = "temporal/client-cert-and-key" +client_key_id = "temporal/client-cert-and-key" +``` + +Rules: + +- A secret ID can't be combined with the credential it replaces, from any source. For example, `api_key` in the file + with `TEMPORAL_EXT_AWS_SECRETS_MANAGER_API_KEY_ID` in the environment is an error. +- The client cert and key IDs must be set together. They may be the same ID, in which case the secret holds both the + PEM certificate and the PEM key. This lets the pair be rotated atomically. +- TLS secret IDs can't be used when TLS is disabled. +- An API key secret enables TLS by default, as `api_key` does. +- Secret values are used as-is: the API key as a string, and TLS material as PEM. Both `SecretString` and + `SecretBinary` secrets are supported. +- As with envconfig's `tls.client_key_{path,data}`, the client key must be PKCS#8 (`-----BEGIN PRIVATE KEY-----`). + Convert other formats with `openssl pkcs8 -topk8 -nocrypt -in key.pem -out key-pkcs8.pem`. +- TLS secret IDs can't be used with an SSL context that you set on the service stubs options yourself. +- Unknown keys under `ext.aws_secrets_manager` are rejected. + +## Fetching and caching + +When `SecretsManagerExtension.Builder.setSecretsClient` isn't used, secrets are fetched through the +[AWS Parameters and Secrets Lambda Extension](https://docs.aws.amazon.com/secretsmanager/latest/userguide/retrieving-secrets_lambda.html) +if it is present. Otherwise they are fetched directly from Secrets Manager with a client built from the default AWS +config. + +The Lambda extension is detected when the config is loaded, as follows: + +- If `PARAMETERS_SECRETS_EXTENSION_HTTP_PORT` is set to a port, the extension is used on that port. +- If it is unset and the code is running in Lambda (`AWS_LAMBDA_FUNCTION_NAME` is set), the extension's default port, + 2773, is probed, and the extension is used if something is listening there. Outside Lambda, the port is not probed + and secrets are fetched directly. +- Setting `PARAMETERS_SECRETS_EXTENSION_HTTP_PORT` to an empty value explicitly disables detection so that secrets are + always fetched directly. Use this if another service in your function listens on port 2773. + +To use your own client, for example with a specific region or credentials, set it on the extension: + +```java +SecretsManagerExtension extension = + SecretsManagerExtension.newBuilder() + .setSecretsClient(SecretsManagerClient.builder().region(Region.US_WEST_2).build()) + .build(); +``` + +Secrets are first fetched when service stubs are created, and creating them fails if a fetch fails. After that: + +- Secrets are cached by the plugin that the extension adds to the loaded profile, so service stubs created with the + same options share the cache (for example, across warm Lambda invocations). +- The API key is refreshed in the background while the service stubs are in use, so a rotated API key is picked up + without creating new stubs. TLS material is fetched only when service stubs are created. +- The refresh interval defaults to the `SECRETS_MANAGER_TTL` environment variable used by the Lambda extension (whole + seconds, where 0 disables caching), or 5 minutes if that's unset. Use `setRefreshInterval` to override it. +- If a refresh fails, the last fetched value keeps being used for up to `setMaxStaleness` (default 24 hours). + +## Observability + +The extension logs with SLF4J under `io.temporal.aws.secretsmanager`, and emits metrics to the service stubs options' +metrics scope. + +TLS secrets are fetched while the service stubs are configured, since the SSL context has to be set on their options, +so their metrics go to the metrics scope already set at that point. Set the metrics scope on the service stubs options +directly, or add a plugin that sets it, like `OpenTelemetryPlugin`, before the plugins from the envconfig profile. +Otherwise, TLS fetches emit no metrics. `OtelLambdaWorkerConfigurationHelper` in `temporal-aws-lambda` already adds its +plugin first. + +Metrics are all labeled with `secret_type` (`api_key`, `tls_client_cert`, `tls_server_ca_cert`; client cert and client +key metrics are both labeled `tls_client_cert`) and `fetch_path` (`extension` or `sdk`). Currently, the following +metrics are emitted: + +| Metric | Type | Extra labels | Description | +|---|---|---|---| +| `temporal_aws_secrets_manager_fetch_failed` | Counter | `failure_reason`, `fallback` (`stale`, `none`) | Failed fetches, and whether the previous value was used instead | +| `temporal_aws_secrets_manager_fetch_latency` | Timer | `outcome` (`success`, `failure`) | Latency of each fetch | +| `temporal_aws_secrets_manager_cache_lookup` | Counter | `result` (`hit`, `miss`, `miss_with_fallback`) | Lookups that returned a value | + +## Required IAM permissions + +The AWS credentials in use (for example, the Lambda execution role) need `secretsmanager:GetSecretValue` on each +configured secret: + +```json +{ + "Effect": "Allow", + "Action": "secretsmanager:GetSecretValue", + "Resource": "arn:aws:secretsmanager:us-west-2:123456789012:secret:temporal/*" +} +``` + +If a secret is encrypted with a customer-managed KMS key, `kms:Decrypt` on that key is also required. diff --git a/contrib/temporal-aws-secrets-manager/build.gradle b/contrib/temporal-aws-secrets-manager/build.gradle new file mode 100644 index 0000000000..4e88bb67f7 --- /dev/null +++ b/contrib/temporal-aws-secrets-manager/build.gradle @@ -0,0 +1,23 @@ +description = '''Temporal Java SDK AWS Secrets Manager Client Config Extension''' + +dependencies { + // This module shouldn't carry temporal-sdk with it, especially for situations when users may + // be using a shaded artifact. + compileOnly project(':temporal-sdk') + + api project(':temporal-envconfig') + implementation "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}" + implementation "org.slf4j:slf4j-api:$slf4jVersion" + + // Optional: only needed to fetch secrets directly from Secrets Manager, when the AWS + // Parameters and Secrets Lambda Extension isn't available. Users add it themselves. + compileOnly platform("software.amazon.awssdk:bom:$awsSdkVersion") + compileOnly "software.amazon.awssdk:secretsmanager" + + testImplementation project(':temporal-sdk') + testImplementation platform("software.amazon.awssdk:bom:$awsSdkVersion") + testImplementation "software.amazon.awssdk:secretsmanager" + testImplementation "junit:junit:${junitVersion}" + testImplementation "org.mockito:mockito-core:${mockitoVersion}" + testRuntimeOnly group: 'ch.qos.logback', name: 'logback-classic', version: "${logbackVersion}" +} diff --git a/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/FetchErrors.java b/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/FetchErrors.java new file mode 100644 index 0000000000..317c821654 --- /dev/null +++ b/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/FetchErrors.java @@ -0,0 +1,47 @@ +package io.temporal.aws.secretsmanager; + +/** Errors from fetching secrets, classified by {@link Telemetry#classifyFailure(Throwable)}. */ +final class FetchErrors { + private FetchErrors() {} + + /** A response that was received but has no usable secret value. */ + static final class InvalidResponseException extends Exception { + InvalidResponseException(String message) { + super(message); + } + + InvalidResponseException(String message, Throwable cause) { + super(message, cause); + } + } + + /** A non-200 response from the Lambda extension. */ + static final class LambdaExtensionStatusException extends Exception { + final int status; + + LambdaExtensionStatusException(int status, String body) { + super("AWS Parameters and Secrets Lambda Extension returned status " + status + ": " + body); + this.status = status; + } + } + + /** A fetch that took longer than the fetch timeout. */ + static final class FetchTimeoutException extends Exception { + FetchTimeoutException(String message, Throwable cause) { + super(message, cause); + } + } + + /** + * An error response from Secrets Manager, with its AWS error code. Wraps the AWS SDK's exception + * so that it can be classified without loading AWS SDK classes. + */ + static final class AwsErrorException extends Exception { + final String errorCode; + + AwsErrorException(String errorCode, Throwable cause) { + super(cause.getMessage(), cause); + this.errorCode = errorCode; + } + } +} diff --git a/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/Fetcher.java b/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/Fetcher.java new file mode 100644 index 0000000000..5889a826f4 --- /dev/null +++ b/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/Fetcher.java @@ -0,0 +1,116 @@ +package io.temporal.aws.secretsmanager; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.nio.charset.StandardCharsets; +import java.util.Map; + +/** Fetches secret values by ID. Must be safe to call from multiple threads. */ +interface Fetcher { + String ENV_LAMBDA_EXTENSION_PORT = "PARAMETERS_SECRETS_EXTENSION_HTTP_PORT"; + String ENV_LAMBDA_FUNCTION_NAME = "AWS_LAMBDA_FUNCTION_NAME"; + String LAMBDA_EXTENSION_HOST = "localhost"; + int DEFAULT_LAMBDA_EXTENSION_PORT = 2773; + + /** Bounds each request made by the fetchers this package creates. */ + long FETCH_TIMEOUT_MILLIS = 10_000; + + /** The fetch_path metric label for this fetcher. */ + String path(); + + /** Fetches the current value of the given secret. */ + SecretValue fetch(String secretId) throws Exception; + + /** A fetched secret value. */ + final class SecretValue { + final byte[] value; + final String versionId; + + SecretValue(byte[] value, String versionId) { + this.value = value; + this.versionId = versionId; + } + + /** A secret value from a GetSecretValue response's fields, preferring SecretString. */ + static SecretValue fromResponse(String secretString, byte[] secretBinary, String versionId) + throws FetchErrors.InvalidResponseException { + byte[] value; + if (secretString != null) { + value = secretString.getBytes(StandardCharsets.UTF_8); + } else if (secretBinary != null && secretBinary.length > 0) { + value = secretBinary; + } else { + throw new FetchErrors.InvalidResponseException("secret has no value"); + } + return new SecretValue(value, versionId != null ? versionId : ""); + } + } + + /** + * Returns a fetcher for the Lambda extension if it is detected, and otherwise an SDK fetcher with + * a client created from the default AWS config. + * + *

The extension doesn't set PARAMETERS_SECRETS_EXTENSION_HTTP_PORT in the function's + * environment unless the user configured it, so if it is unset and running in Lambda, the + * extension's default port is probed. Setting it to empty disables detection, for when another + * service uses the default port. + */ + static Fetcher newDefaultFetcher(Map processEnv) { + String port = processEnv.get(ENV_LAMBDA_EXTENSION_PORT); + if (port == null) { + // Only try to auto-detect the Lambda extension if we're in a Lambda function + String functionName = processEnv.get(ENV_LAMBDA_FUNCTION_NAME); + if (functionName == null + || functionName.isEmpty() + || !probe(LAMBDA_EXTENSION_HOST, DEFAULT_LAMBDA_EXTENSION_PORT)) { + return newDefaultSdkFetcher(); + } + port = String.valueOf(DEFAULT_LAMBDA_EXTENSION_PORT); + } + if (port.isEmpty()) { + return newDefaultSdkFetcher(); + } + return new LambdaExtensionFetcher(port, processEnv); + } + + /** Whether something accepts connections on the given port. */ + static boolean probe(String host, int port) { + // Probing a local port is either accepted or refused almost immediately, so the timeout only + // bounds pathological cases + try (Socket socket = new Socket()) { + socket.connect(new InetSocketAddress(host, port), 100); + return true; + } catch (IOException e) { + return false; + } + } + + /** + * Whether the AWS SDK Secrets Manager client can be loaded. Checked without loading {@link + * SdkFetcher}, which references AWS SDK classes. + */ + static boolean isAwsSdkAvailable() { + try { + Class.forName( + "software.amazon.awssdk.services.secretsmanager.SecretsManagerClient", + false, + Fetcher.class.getClassLoader()); + return true; + } catch (ClassNotFoundException | LinkageError e) { + return false; + } + } + + static Fetcher newDefaultSdkFetcher() { + if (!isAwsSdkAvailable()) { + throw new IllegalStateException( + "fetching secrets directly from AWS Secrets Manager requires the AWS SDK, since the AWS " + + "Parameters and Secrets Lambda Extension was not detected (see " + + ENV_LAMBDA_EXTENSION_PORT + + "). Add a dependency on software.amazon.awssdk:secretsmanager, or add the extension " + + "to your Lambda function"); + } + return SdkFetcher.create(null); + } +} diff --git a/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/LambdaExtensionFetcher.java b/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/LambdaExtensionFetcher.java new file mode 100644 index 0000000000..6385274cd0 --- /dev/null +++ b/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/LambdaExtensionFetcher.java @@ -0,0 +1,129 @@ +package io.temporal.aws.secretsmanager; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.net.HttpURLConnection; +import java.net.Proxy; +import java.net.SocketTimeoutException; +import java.net.URL; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.Map; + +/** + * Fetches secrets through the AWS Parameters and Secrets Lambda Extension. See + * https://docs.aws.amazon.com/secretsmanager/latest/userguide/retrieving-secrets_lambda.html. + */ +final class LambdaExtensionFetcher implements Fetcher { + private static final String ENV_AWS_SESSION_TOKEN = "AWS_SESSION_TOKEN"; + private static final String TOKEN_HEADER = "X-Aws-Parameters-Secrets-Token"; + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private final String endpoint; + private final Map processEnv; + + LambdaExtensionFetcher(String port, Map processEnv) { + this.endpoint = + "http://" + LAMBDA_EXTENSION_HOST + ":" + port + "/secretsmanager/get?secretId="; + this.processEnv = processEnv; + } + + @Override + public String path() { + return Telemetry.FETCH_PATH_EXTENSION; + } + + @Override + public SecretValue fetch(String secretId) throws Exception { + URL url = new URL(endpoint + urlEncode(secretId)); + // No proxy, so that proxy settings never apply to requests to the local extension + HttpURLConnection conn = (HttpURLConnection) url.openConnection(Proxy.NO_PROXY); + conn.setConnectTimeout((int) FETCH_TIMEOUT_MILLIS); + conn.setReadTimeout((int) FETCH_TIMEOUT_MILLIS); + String token = processEnv.get(ENV_AWS_SESSION_TOKEN); + conn.setRequestProperty(TOKEN_HEADER, token != null ? token : ""); + int status; + String body; + try { + status = conn.getResponseCode(); + body = readAll(status >= 400 ? conn.getErrorStream() : conn.getInputStream()); + } catch (SocketTimeoutException e) { + throw new FetchErrors.FetchTimeoutException( + "AWS Secrets Manager secret fetch timed out after " + FETCH_TIMEOUT_MILLIS + "ms", e); + } + if (status != 200) { + throw new FetchErrors.LambdaExtensionStatusException(status, body); + } + return parseResponse(body); + } + + static SecretValue parseResponse(String body) throws FetchErrors.InvalidResponseException { + JsonNode out; + try { + // Same shape as the GetSecretValue response, with SecretBinary base64 + out = MAPPER.readTree(body); + } catch (IOException e) { + throw invalid(e.getMessage(), e); + } + if (out == null || !out.isObject()) { + throw invalid("not an object", null); + } + String secretString = textField(out, "SecretString"); + String secretBinary = textField(out, "SecretBinary"); + String versionId = textField(out, "VersionId"); + byte[] binary = null; + if (secretBinary != null && !secretBinary.isEmpty()) { + try { + binary = Base64.getDecoder().decode(secretBinary); + } catch (IllegalArgumentException e) { + throw invalid("invalid SecretBinary: " + e.getMessage(), e); + } + } + return SecretValue.fromResponse(secretString, binary, versionId); + } + + private static String textField(JsonNode out, String name) + throws FetchErrors.InvalidResponseException { + JsonNode field = out.get(name); + if (field == null || field.isNull()) { + return null; + } + if (!field.isTextual()) { + throw invalid(name + " must be a string", null); + } + return field.asText(); + } + + private static FetchErrors.InvalidResponseException invalid(String message, Throwable cause) { + return new FetchErrors.InvalidResponseException( + "failed decoding AWS Parameters and Secrets Lambda Extension response: " + message, cause); + } + + private static String urlEncode(String s) { + try { + return URLEncoder.encode(s, "UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new IllegalStateException(e); + } + } + + private static String readAll(InputStream in) throws IOException { + if (in == null) { + return ""; + } + try (InputStream stream = in) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buf = new byte[8192]; + int n; + while ((n = stream.read(buf)) != -1) { + out.write(buf, 0, n); + } + return new String(out.toByteArray(), StandardCharsets.UTF_8); + } + } +} diff --git a/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/SdkFetcher.java b/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/SdkFetcher.java new file mode 100644 index 0000000000..dbb8c7a867 --- /dev/null +++ b/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/SdkFetcher.java @@ -0,0 +1,76 @@ +package io.temporal.aws.secretsmanager; + +import java.time.Duration; +import javax.annotation.Nullable; +import software.amazon.awssdk.awscore.AwsRequestOverrideConfiguration; +import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.core.SdkBytes; +import software.amazon.awssdk.core.exception.ApiCallAttemptTimeoutException; +import software.amazon.awssdk.core.exception.ApiCallTimeoutException; +import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient; +import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueRequest; +import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueResponse; + +/** + * Fetches secrets directly from Secrets Manager with the AWS SDK, which is an optional dependency. + * This is the only class that references AWS SDK classes, so it must only be loaded when the AWS + * SDK is available. + */ +final class SdkFetcher implements Fetcher { + private static final AwsRequestOverrideConfiguration TIMEOUT = + AwsRequestOverrideConfiguration.builder() + .apiCallTimeout(Duration.ofMillis(FETCH_TIMEOUT_MILLIS)) + .build(); + + private final Object lock = new Object(); + private SecretsManagerClient client; + + private SdkFetcher(@Nullable SecretsManagerClient client) { + this.client = client; + } + + /** + * Creates a fetcher with the given client, which must be a {@code SecretsManagerClient}, or with + * a client created from the default AWS config on first use if null. + */ + static SdkFetcher create(@Nullable Object client) { + return new SdkFetcher((SecretsManagerClient) client); + } + + @Override + public String path() { + return Telemetry.FETCH_PATH_SDK; + } + + @Override + public SecretValue fetch(String secretId) throws Exception { + GetSecretValueResponse resp; + try { + resp = + getClient() + .getSecretValue( + GetSecretValueRequest.builder() + .secretId(secretId) + .overrideConfiguration(TIMEOUT) + .build()); + } catch (ApiCallTimeoutException | ApiCallAttemptTimeoutException e) { + throw new FetchErrors.FetchTimeoutException( + "AWS Secrets Manager secret fetch timed out after " + FETCH_TIMEOUT_MILLIS + "ms", e); + } catch (AwsServiceException e) { + throw new FetchErrors.AwsErrorException( + e.awsErrorDetails() != null ? e.awsErrorDetails().errorCode() : null, e); + } + SdkBytes binary = resp.secretBinary(); + return SecretValue.fromResponse( + resp.secretString(), binary != null ? binary.asByteArray() : null, resp.versionId()); + } + + private SecretsManagerClient getClient() { + synchronized (lock) { + if (client == null) { + client = SecretsManagerClient.create(); + } + return client; + } + } +} diff --git a/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/SecretCache.java b/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/SecretCache.java new file mode 100644 index 0000000000..7c78a84650 --- /dev/null +++ b/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/SecretCache.java @@ -0,0 +1,646 @@ +package io.temporal.aws.secretsmanager; + +import java.io.ByteArrayInputStream; +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.security.GeneralSecurityException; +import java.security.KeyFactory; +import java.security.PrivateKey; +import java.security.Signature; +import java.security.cert.CertificateFactory; +import java.security.cert.X509Certificate; +import java.security.spec.PKCS8EncodedKeySpec; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.LongSupplier; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import javax.annotation.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * TTL cache of the configured credentials that falls back to the last fetched value, up to the max + * staleness old, when a refresh fails. + * + *

Fetches run on a shared pool of daemon threads, so that they aren't affected by callers timing + * out or being interrupted, and don't keep the JVM running. + */ +final class SecretCache { + private static final Logger log = LoggerFactory.getLogger(SecretCache.class); + + /** + * After a failed refresh falls back to a stale value, wait this long before trying again so that + * an outage doesn't add a fetch attempt to every credential lookup. + */ + static final long FAILED_REFRESH_RETRY_INTERVAL_NANOS = TimeUnit.SECONDS.toNanos(30); + + private static final Pattern CERT_PEM_PATTERN = + Pattern.compile( + "-----BEGIN CERTIFICATE-----\r?\n.*?-----END CERTIFICATE-----", Pattern.DOTALL); + private static final Pattern KEY_PEM_PATTERN = + Pattern.compile( + "-----BEGIN ((?:RSA |EC )?PRIVATE KEY)-----\r?\n(.*?)-----END \\1-----", Pattern.DOTALL); + // Key algorithms that PKCS#8 keys are tried as, matching Netty's key loading used by envconfig + private static final List KEY_ALGORITHMS = Arrays.asList("RSA", "DSA", "EC"); + + private static final AtomicInteger FETCH_THREAD_COUNT = new AtomicInteger(); + private static final ExecutorService FETCH_EXECUTOR = + Executors.newCachedThreadPool( + r -> { + Thread t = + new Thread( + r, + "temporal-aws-secrets-manager-fetch-" + FETCH_THREAD_COUNT.incrementAndGet()); + t.setDaemon(true); + return t; + }); + + /** Configured secret IDs. Empty strings are unset. */ + static final class SecretIds { + final String apiKey; + final String clientCert; + final String clientKey; + final String serverCaCert; + + SecretIds(String apiKey, String clientCert, String clientKey, String serverCaCert) { + this.apiKey = apiKey; + this.clientCert = clientCert; + this.clientKey = clientKey; + this.serverCaCert = serverCaCert; + } + + boolean hasTls() { + return !clientCert.isEmpty() || !clientKey.isEmpty() || !serverCaCert.isEmpty(); + } + + boolean isEmpty() { + return apiKey.isEmpty() && !hasTls(); + } + } + + /** A client cert chain and its private key. */ + static final class ClientCertPair { + final X509Certificate[] chain; + final PrivateKey key; + + ClientCertPair(X509Certificate[] chain, PrivateKey key) { + this.chain = chain; + this.key = key; + } + } + + /** A failed fetch of one or more secrets, with its failure_reason label. */ + static final class SecretFetchException extends RuntimeException { + final String reason; + + SecretFetchException(String message, String reason, @Nullable Throwable cause) { + super(message, cause); + this.reason = reason; + } + } + + /** Thrown when waiting for a fetch takes longer than the fetch timeout. */ + static final class FetchWaitTimeoutException extends RuntimeException { + FetchWaitTimeoutException(String message) { + super(message); + } + } + + @FunctionalInterface + interface Parser { + /** Parses newly fetched secrets, in refs order, into the cached value. */ + T parse(List secrets) throws FetchErrors.InvalidResponseException; + } + + static final class SecretRef { + final String id; + final String secretType; + + SecretRef(String id, String secretType) { + this.id = id; + this.secretType = secretType; + } + } + + /** + * One or more secrets that are fetched and replaced together, cached as a single parsed value. + */ + static final class CacheEntry { + /** Distinct secret IDs. refs[0]'s secret type is used as the secret_type metric label. */ + final List refs; + + final Parser parser; + + // All fields below are guarded by this + + /** The fetch in progress, or null if none. */ + CompletableFuture inFlight; + + /** In refs order, null until the first successful fetch. */ + List versionIds; + + T value; + long fetchedAt; + + /** Last failed fetch since the last successful fetch, or null if none. */ + Long failedAt; + + CacheEntry(List refs, Parser parser) { + this.refs = refs; + this.parser = parser; + } + } + + final Fetcher fetcher; + final long refreshIntervalNanos; + final long maxStalenessNanos; + + /** How long callers wait for a fetch before falling back or failing. */ + volatile long fetchTimeoutNanos = TimeUnit.MILLISECONDS.toNanos(Fetcher.FETCH_TIMEOUT_MILLIS); + + /** Monotonic nanoseconds, overridable in tests. */ + volatile LongSupplier now = System::nanoTime; + + /** Runs fetches, overridable in tests. */ + volatile Executor executor = FETCH_EXECUTOR; + + @Nullable final CacheEntry apiKey; + + /** Client cert and key, fetched and replaced together to prevent tearing on rotations. */ + @Nullable final CacheEntry clientCert; + + @Nullable final CacheEntry serverCaCert; + + SecretCache(Fetcher fetcher, SecretIds ids, long refreshIntervalNanos, long maxStalenessNanos) { + this.fetcher = fetcher; + this.refreshIntervalNanos = refreshIntervalNanos; + this.maxStalenessNanos = maxStalenessNanos; + this.apiKey = + ids.apiKey.isEmpty() + ? null + : new CacheEntry<>( + Collections.singletonList(new SecretRef(ids.apiKey, Telemetry.SECRET_TYPE_API_KEY)), + SecretCache::parseApiKey); + this.clientCert = + ids.clientCert.isEmpty() ? null : newClientCertEntry(ids.clientCert, ids.clientKey); + this.serverCaCert = ids.serverCaCert.isEmpty() ? null : newServerCaCertEntry(ids.serverCaCert); + } + + /** + * Gets an entry's value, fetching it if needed. Returned values are shared across callers and + * must not be mutated. + * + * @throws SecretFetchException if the fetch failed and there's no usable previous value + * @throws FetchWaitTimeoutException if the fetch timed out and there's no usable previous value + */ + T get(CacheEntry entry, Telemetry tel) { + long start; + boolean canUseStale; + T cached; + CompletableFuture fetch = null; + boolean startFetch = false; + String result = null; + synchronized (entry) { + start = now.getAsLong(); + boolean hasValue = entry.versionIds != null; + canUseStale = hasValue && start - entry.fetchedAt < maxStalenessNanos; + cached = entry.value; + if ((hasValue && start - entry.fetchedAt < refreshIntervalNanos) + // Another caller is refreshing, so the current value is still the latest known one + || (canUseStale && entry.inFlight != null)) { + result = "hit"; + } else if (canUseStale + && entry.failedAt != null + && start - entry.failedAt < FAILED_REFRESH_RETRY_INTERVAL_NANOS) { + result = "miss_with_fallback"; + } else { + // Wait for a fetch, starting one if none is in progress + fetch = entry.inFlight; + if (fetch == null) { + fetch = new CompletableFuture<>(); + entry.inFlight = fetch; + startFetch = true; + } + } + } + if (result != null) { + recordLookup(entry, tel, result); + return cached; + } + if (startFetch) { + startFetch(entry, fetch, start, canUseStale, tel); + } + + T value; + try { + value = fetch.get(fetchTimeoutNanos, TimeUnit.NANOSECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + if (canUseStale) { + recordLookup(entry, tel, "miss_with_fallback"); + return cached; + } + throw new FetchWaitTimeoutException( + "interrupted while waiting for AWS Secrets Manager secret fetch"); + } catch (TimeoutException e) { + if (canUseStale) { + recordLookup(entry, tel, "miss_with_fallback"); + return cached; + } + throw new FetchWaitTimeoutException( + "timed out after " + + TimeUnit.NANOSECONDS.toMillis(fetchTimeoutNanos) + + "ms waiting for AWS Secrets Manager secret fetch"); + } catch (ExecutionException e) { + if (canUseStale) { + recordLookup(entry, tel, "miss_with_fallback"); + return cached; + } + Throwable cause = e.getCause(); + if (cause instanceof SecretFetchException) { + throw (SecretFetchException) cause; + } + throw new SecretFetchException(String.valueOf(cause), "other", cause); + } + recordLookup(entry, tel, "miss"); + return value; + } + + /** + * Returns an entry's current value without waiting, and starts refreshing it in the background if + * it's due. The entry must already have a value. + * + *

The value is refreshed at most every {@code minRefreshIntervalNanos}, and is kept if + * refreshing it fails, even past the max staleness. + */ + T getNow(CacheEntry entry, long minRefreshIntervalNanos, Telemetry tel) { + long start; + boolean canUseStale; + T value; + CompletableFuture fetch = null; + synchronized (entry) { + start = now.getAsLong(); + value = entry.value; + canUseStale = entry.versionIds != null && start - entry.fetchedAt < maxStalenessNanos; + if (entry.inFlight == null + && start - entry.fetchedAt >= Math.max(refreshIntervalNanos, minRefreshIntervalNanos) + && (entry.failedAt == null + || start - entry.failedAt >= FAILED_REFRESH_RETRY_INTERVAL_NANOS)) { + fetch = new CompletableFuture<>(); + entry.inFlight = fetch; + } + } + if (fetch != null) { + startFetch(entry, fetch, start, canUseStale, tel); + } + return value; + } + + private void startFetch( + CacheEntry entry, + CompletableFuture fetch, + long start, + boolean canUseStale, + Telemetry tel) { + try { + executor.execute(() -> refresh(entry, fetch, start, canUseStale, tel)); + } catch (RejectedExecutionException e) { + // Fail this fetch, including for callers already waiting on it, so that the next caller + // starts a new one + synchronized (entry) { + entry.inFlight = null; + } + fetch.completeExceptionally( + new SecretFetchException("failed starting AWS Secrets Manager secret fetch", "other", e)); + } + } + + /** Fetches an entry's secrets and records the outcome in the cache. */ + private void refresh( + CacheEntry entry, + CompletableFuture fetch, + long start, + boolean canUseStale, + Telemetry tel) { + T value = null; + List versionIds = null; + SecretFetchException failure = null; + try { + List secrets = fetchAll(entry); + value = parse(entry, secrets); + versionIds = new ArrayList<>(secrets.size()); + for (Fetcher.SecretValue secret : secrets) { + versionIds.add(secret.versionId); + } + } catch (SecretFetchException e) { + failure = e; + } catch (RuntimeException e) { + // Shouldn't happen, but don't leave the fetch pending + failure = new SecretFetchException(e.toString(), "other", e); + } + long latencyNanos = now.getAsLong() - start; + + List prevVersionIds; + long prevFetchedAt; + boolean prevFailed; + synchronized (entry) { + prevVersionIds = entry.versionIds; + prevFetchedAt = entry.fetchedAt; + prevFailed = entry.failedAt != null; + if (failure != null) { + entry.failedAt = start; + } else { + entry.value = value; + entry.versionIds = versionIds; + entry.fetchedAt = start; + entry.failedAt = null; + } + entry.inFlight = null; + } + + // Record telemetry outside the lock, and complete the fetch last so that the fetch's metrics + // and logs are recorded by the time waiters return + try { + if (failure != null) { + recordFailure(entry, failure, latencyNanos, start - prevFetchedAt, canUseStale, tel); + } else { + recordSuccess(entry, versionIds, prevVersionIds, prevFailed, latencyNanos, tel); + } + } catch (RuntimeException e) { + log.warn("Failed recording AWS Secrets Manager telemetry", e); + } finally { + if (failure != null) { + fetch.completeExceptionally(failure); + } else { + fetch.complete(value); + } + } + } + + private List fetchAll(CacheEntry entry) { + List secrets = new ArrayList<>(entry.refs.size()); + for (SecretRef ref : entry.refs) { + try { + secrets.add(fetcher.fetch(ref.id)); + } catch (Exception e) { + throw new SecretFetchException( + "failed fetching AWS Secrets Manager secret \"" + + ref.id + + "\": " + + (e.getMessage() != null ? e.getMessage() : e.toString()), + Telemetry.classifyFailure(e), + e); + } + } + return secrets; + } + + private static T parse(CacheEntry entry, List secrets) { + try { + return entry.parser.parse(secrets); + } catch (FetchErrors.InvalidResponseException e) { + throw new SecretFetchException(e.getMessage(), "invalid_response", e); + } + } + + private Map baseTags(CacheEntry entry) { + Map tags = new HashMap<>(); + tags.put(Telemetry.LABEL_SECRET_TYPE, entry.refs.get(0).secretType); + tags.put(Telemetry.LABEL_FETCH_PATH, fetcher.path()); + return tags; + } + + private void recordLookup(CacheEntry entry, Telemetry tel, String result) { + Map tags = baseTags(entry); + tags.put(Telemetry.LABEL_RESULT, result); + tel.cacheLookup(tags); + } + + private void recordFailure( + CacheEntry entry, + SecretFetchException err, + long latencyNanos, + long ageNanos, + boolean canUseStale, + Telemetry tel) { + Map latencyTags = baseTags(entry); + latencyTags.put(Telemetry.LABEL_OUTCOME, "failure"); + tel.fetchLatency(latencyTags, latencyNanos); + Map failedTags = baseTags(entry); + failedTags.put(Telemetry.LABEL_FAILURE_REASON, err.reason); + failedTags.put(Telemetry.LABEL_FALLBACK, canUseStale ? "stale" : "none"); + tel.fetchFailed(failedTags); + if (canUseStale) { + for (SecretRef ref : entry.refs) { + log.warn( + "Failed refreshing AWS Secrets Manager secret, using previously fetched value " + + "(secret_id={}, secret_type={}, error={}, age={}, max_staleness={})", + ref.id, + ref.secretType, + err.getMessage(), + Duration.ofNanos(ageNanos), + Duration.ofNanos(maxStalenessNanos)); + } + } + } + + private void recordSuccess( + CacheEntry entry, + List versionIds, + @Nullable List prevVersionIds, + boolean prevFailed, + long latencyNanos, + Telemetry tel) { + Map tags = baseTags(entry); + tags.put(Telemetry.LABEL_OUTCOME, "success"); + tel.fetchLatency(tags, latencyNanos); + for (int i = 0; i < entry.refs.size(); i++) { + SecretRef ref = entry.refs.get(i); + if (prevVersionIds == null) { + log.info( + "Fetched AWS Secrets Manager secret " + + "(secret_id={}, secret_type={}, fetch_path={}, version_id={})", + ref.id, + ref.secretType, + fetcher.path(), + versionIds.get(i)); + } else if (prevFailed || !versionIds.get(i).equals(prevVersionIds.get(i))) { + log.info( + "Fetched updated AWS Secrets Manager secret " + + "(secret_id={}, secret_type={}, previous_version_id={}, version_id={})", + ref.id, + ref.secretType, + prevVersionIds.get(i), + versionIds.get(i)); + } + } + } + + private static String parseApiKey(List secrets) + throws FetchErrors.InvalidResponseException { + try { + return StandardCharsets.UTF_8 + .newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(secrets.get(0).value)) + .toString(); + } catch (CharacterCodingException e) { + throw new FetchErrors.InvalidResponseException("API key secret is not valid UTF-8"); + } + } + + /** + * Entry for the client cert and key, which may be the same secret if bundled. Each secret's + * relevant PEM blocks are extracted, so a bundled secret can be used as both. Keys must be + * PKCS#8, like keys in envconfig. A newly fetched pair that doesn't match (e.g. fetched partway + * through rotating separate cert and key secrets) is treated as a failed fetch, so the previous + * pair keeps being used. + */ + private static CacheEntry newClientCertEntry(String certId, String keyId) { + List refs = new ArrayList<>(); + refs.add(new SecretRef(certId, Telemetry.SECRET_TYPE_TLS_CLIENT_CERT)); + if (!keyId.equals(certId)) { + refs.add(new SecretRef(keyId, Telemetry.SECRET_TYPE_TLS_CLIENT_KEY)); + } + return new CacheEntry<>( + refs, + secrets -> { + X509Certificate[] chain = parseCerts(secrets.get(0).value, "TLS client cert", certId); + if (chain.length == 0) { + throw new FetchErrors.InvalidResponseException( + "no PEM certificate found in TLS client cert secret \"" + certId + "\""); + } + PrivateKey key = parseKey(secrets.get(secrets.size() - 1).value, keyId); + if (!keyMatchesCert(key, chain[0])) { + throw new FetchErrors.InvalidResponseException( + "TLS client cert and key from AWS Secrets Manager secrets \"" + + certId + + "\" and \"" + + keyId + + "\" don't match"); + } + return new ClientCertPair(chain, key); + }); + } + + private static CacheEntry newServerCaCertEntry(String secretId) { + return new CacheEntry<>( + Collections.singletonList( + new SecretRef(secretId, Telemetry.SECRET_TYPE_TLS_SERVER_CA_CERT)), + secrets -> { + X509Certificate[] certs = + parseCerts(secrets.get(0).value, "TLS server CA cert", secretId); + if (certs.length == 0) { + throw new FetchErrors.InvalidResponseException( + "no PEM certificates found in TLS server CA cert secret \"" + secretId + "\""); + } + return certs; + }); + } + + private static X509Certificate[] parseCerts(byte[] secret, String what, String secretId) + throws FetchErrors.InvalidResponseException { + Matcher m = CERT_PEM_PATTERN.matcher(new String(secret, StandardCharsets.UTF_8)); + List certs = new ArrayList<>(); + try { + CertificateFactory factory = CertificateFactory.getInstance("X.509"); + while (m.find()) { + certs.add( + (X509Certificate) + factory.generateCertificate( + new ByteArrayInputStream(m.group().getBytes(StandardCharsets.US_ASCII)))); + } + } catch (GeneralSecurityException e) { + throw new FetchErrors.InvalidResponseException( + "invalid PEM certificate in " + what + " secret \"" + secretId + "\": " + e.getMessage(), + e); + } + return certs.toArray(new X509Certificate[0]); + } + + private static PrivateKey parseKey(byte[] secret, String keyId) + throws FetchErrors.InvalidResponseException { + Matcher m = KEY_PEM_PATTERN.matcher(new String(secret, StandardCharsets.UTF_8)); + if (!m.find()) { + throw new FetchErrors.InvalidResponseException( + "no PEM private key found in TLS client key secret \"" + keyId + "\""); + } + if (!m.group(1).equals("PRIVATE KEY")) { + throw new FetchErrors.InvalidResponseException( + "TLS client key secret \"" + + keyId + + "\" is not a PKCS#8 key: its PEM header is \"-----BEGIN " + + m.group(1) + + "-----\" instead of \"-----BEGIN PRIVATE KEY-----\". Convert it with: openssl " + + "pkcs8 -topk8 -nocrypt -in key.pem -out key-pkcs8.pem"); + } + byte[] der; + try { + der = Base64.getMimeDecoder().decode(m.group(2)); + } catch (IllegalArgumentException e) { + throw new FetchErrors.InvalidResponseException( + "invalid PEM private key in TLS client key secret \"" + keyId + "\"", e); + } + PKCS8EncodedKeySpec spec = new PKCS8EncodedKeySpec(der); + for (String algorithm : KEY_ALGORITHMS) { + try { + return KeyFactory.getInstance(algorithm).generatePrivate(spec); + } catch (GeneralSecurityException e) { + // Try the next algorithm + } + } + throw new FetchErrors.InvalidResponseException( + "invalid PEM private key in TLS client key secret \"" + keyId + "\""); + } + + /** Whether a private key matches a cert's public key, checked by signing and verifying. */ + private static boolean keyMatchesCert(PrivateKey key, X509Certificate cert) { + String algorithm; + switch (key.getAlgorithm()) { + case "RSA": + algorithm = "SHA256withRSA"; + break; + case "EC": + algorithm = "SHA256withECDSA"; + break; + case "DSA": + algorithm = "SHA256withDSA"; + break; + default: + return false; + } + byte[] data = "temporal-aws-secrets-manager".getBytes(StandardCharsets.US_ASCII); + try { + Signature signer = Signature.getInstance(algorithm); + signer.initSign(key); + signer.update(data); + byte[] signature = signer.sign(); + Signature verifier = Signature.getInstance(algorithm); + verifier.initVerify(cert.getPublicKey()); + verifier.update(data); + return verifier.verify(signature); + } catch (GeneralSecurityException e) { + // e.g. an EC key with an RSA cert + return false; + } + } +} diff --git a/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/SecretsManagerExtension.java b/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/SecretsManagerExtension.java new file mode 100644 index 0000000000..aab5739a68 --- /dev/null +++ b/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/SecretsManagerExtension.java @@ -0,0 +1,352 @@ +package io.temporal.aws.secretsmanager; + +import io.temporal.common.Experimental; +import io.temporal.envconfig.ClientConfigExtension; +import io.temporal.envconfig.ClientConfigExtensionRequest; +import io.temporal.envconfig.ClientConfigProfile; +import io.temporal.envconfig.ClientConfigTLS; +import io.temporal.serviceclient.WorkflowServiceStubsPlugin; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.regex.Pattern; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient; + +/** + * Client config extension that loads client credentials (API key and/or mTLS material) from AWS + * Secrets Manager. + * + *

Pass to {@link + * io.temporal.envconfig.LoadClientConfigProfileOptions.Builder#setExtensions(List)}: + * + *

{@code
+ * ClientConfigProfile profile =
+ *     ClientConfigProfile.load(
+ *         LoadClientConfigProfileOptions.newBuilder()
+ *             .setExtensions(Collections.singletonList(SecretsManagerExtension.getDefaultInstance()))
+ *             .build());
+ * WorkflowServiceStubs service =
+ *     WorkflowServiceStubs.newServiceStubs(profile.toWorkflowServiceStubsOptions());
+ * }
+ * + *

Secret IDs are configured in the {@code profile..ext.aws_secrets_manager} TOML table + * or with {@code TEMPORAL_EXT_AWS_SECRETS_MANAGER_*} environment variables. See the module README + * for details. + */ +@Experimental +public final class SecretsManagerExtension implements ClientConfigExtension { + /** The name of the extension, which is also its table key under {@code profile..ext}. */ + public static final String EXTENSION_NAME = "aws_secrets_manager"; + + private static final String ENV_API_KEY_ID = "TEMPORAL_EXT_AWS_SECRETS_MANAGER_API_KEY_ID"; + private static final String ENV_CLIENT_CERT_ID = + "TEMPORAL_EXT_AWS_SECRETS_MANAGER_TLS_CLIENT_CERT_ID"; + private static final String ENV_CLIENT_KEY_ID = + "TEMPORAL_EXT_AWS_SECRETS_MANAGER_TLS_CLIENT_KEY_ID"; + private static final String ENV_SERVER_CA_CERT_ID = + "TEMPORAL_EXT_AWS_SECRETS_MANAGER_TLS_SERVER_CA_CERT_ID"; + + // Same env var and default as the AWS Parameters and Secrets Lambda Extension's cache TTL + private static final String ENV_SECRETS_MANAGER_TTL = "SECRETS_MANAGER_TTL"; + private static final Pattern WHOLE_NUMBER = Pattern.compile("[0-9]+"); + private static final Duration DEFAULT_REFRESH_INTERVAL = Duration.ofMinutes(5); + private static final Duration DEFAULT_MAX_STALENESS = Duration.ofHours(24); + + private static final SecretsManagerExtension DEFAULT_INSTANCE = newBuilder().build(); + + /** Returns an extension with the default options. */ + public static SecretsManagerExtension getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + /** Creates a builder for an extension with non-default options. */ + public static Builder newBuilder() { + return new Builder(); + } + + // A SecretsManagerClient, held as an Object so that this class can be loaded without the AWS SDK + @Nullable private final Object secretsClient; + @Nullable private final Duration refreshInterval; + @Nullable private final Duration maxStaleness; + private final Map processEnv; + + private SecretsManagerExtension(Builder builder) { + this.secretsClient = builder.secretsClient; + this.refreshInterval = builder.refreshInterval; + this.maxStaleness = builder.maxStaleness; + this.processEnv = builder.processEnv; + } + + /** The extension name, {@code aws_secrets_manager}. */ + @Nonnull + @Override + public String getName() { + return EXTENSION_NAME; + } + + /** + * Reads secret IDs from the profile and env, and returns a plugin that fetches the configured + * secrets, or null if none are configured. + */ + @Nullable + @Override + public WorkflowServiceStubsPlugin getClientPlugin(@Nonnull ClientConfigExtensionRequest request) { + ClientConfigProfile profile = request.getProfile(); + SecretCache.SecretIds ids = loadSecretIds(profile, request.getEnv()); + if (ids.isEmpty()) { + return null; + } + + Duration refresh = refreshInterval(); + Duration staleness = maxStaleness; + if (staleness == null || staleness.isZero() || staleness.isNegative()) { + staleness = DEFAULT_MAX_STALENESS; + } + if (staleness.compareTo(refresh) < 0) { + staleness = refresh; + } + + Fetcher fetcher = + secretsClient != null + ? SdkFetcher.create(secretsClient) + : Fetcher.newDefaultFetcher(processEnv); + SecretCache cache = new SecretCache(fetcher, ids, nanos(refresh), nanos(staleness)); + ClientConfigTLS tls = profile.getTls(); + return new SecretsManagerPlugin( + ids, cache, tls != null && !Boolean.TRUE.equals(tls.isDisabled()) ? tls : null); + } + + private Duration refreshInterval() { + if (refreshInterval != null) { + if (refreshInterval.isNegative()) { + throw new IllegalArgumentException("refreshInterval must not be negative"); + } + return refreshInterval; + } + // Read from the process env rather than the config env, like + // PARAMETERS_SECRETS_EXTENSION_HTTP_PORT, since this is Lambda runtime config shared with the + // Lambda extension rather than Temporal client config + String ttl = processEnv.get(ENV_SECRETS_MANAGER_TTL); + if (ttl == null || ttl.isEmpty()) { + return DEFAULT_REFRESH_INTERVAL; + } + if (WHOLE_NUMBER.matcher(ttl).matches()) { + try { + return Duration.ofSeconds(Long.parseLong(ttl)); + } catch (NumberFormatException e) { + // Too large, handled below + } + } + throw new IllegalArgumentException( + "invalid " + + ENV_SECRETS_MANAGER_TTL + + " \"" + + ttl + + "\", must be a non-negative whole number of seconds"); + } + + /** Nanoseconds, saturating rather than overflowing. */ + private static long nanos(Duration d) { + try { + return d.toNanos(); + } catch (ArithmeticException e) { + return Long.MAX_VALUE; + } + } + + private static SecretCache.SecretIds loadSecretIds( + ClientConfigProfile profile, @Nullable Map env) { + Map fields = new HashMap<>(); + applyToml(fields, profile.getExt().get(EXTENSION_NAME)); + if (env != null) { + // Like envconfig, set-but-empty env vars override (and so can unset) file values + putEnv(fields, env, ENV_API_KEY_ID, "api_key"); + putEnv(fields, env, ENV_CLIENT_CERT_ID, "client_cert"); + putEnv(fields, env, ENV_CLIENT_KEY_ID, "client_key"); + putEnv(fields, env, ENV_SERVER_CA_CERT_ID, "server_ca_cert"); + } + SecretCache.SecretIds ids = + new SecretCache.SecretIds( + fields.getOrDefault("api_key", ""), + fields.getOrDefault("client_cert", ""), + fields.getOrDefault("client_key", ""), + fields.getOrDefault("server_ca_cert", "")); + validate(ids, profile); + return ids; + } + + private static void putEnv( + Map fields, Map env, String name, String field) { + String value = env.get(name); + if (value != null) { + fields.put(field, value); + } + } + + private static void applyToml(Map fields, @Nullable Map table) { + if (table == null) { + return; + } + for (Map.Entry e : table.entrySet()) { + String key = e.getKey(); + if (key.equals("api_key_id")) { + fields.put("api_key", tomlString(key, e.getValue())); + } else if (key.equals("tls")) { + if (!(e.getValue() instanceof Map)) { + throw new IllegalArgumentException( + "ext." + EXTENSION_NAME + ".tls must be a table, got " + typeName(e.getValue())); + } + for (Map.Entry tlsEntry : ((Map) e.getValue()).entrySet()) { + String tlsKey = String.valueOf(tlsEntry.getKey()); + String field; + switch (tlsKey) { + case "client_cert_id": + field = "client_cert"; + break; + case "client_key_id": + field = "client_key"; + break; + case "server_ca_cert_id": + field = "server_ca_cert"; + break; + default: + throw new IllegalArgumentException( + "unknown key \"" + tlsKey + "\" in ext." + EXTENSION_NAME + ".tls"); + } + fields.put(field, tomlString("tls." + tlsKey, tlsEntry.getValue())); + } + } else { + throw new IllegalArgumentException("unknown key \"" + key + "\" in ext." + EXTENSION_NAME); + } + } + } + + private static String tomlString(String key, Object value) { + if (!(value instanceof String)) { + throw new IllegalArgumentException( + "ext." + EXTENSION_NAME + "." + key + " must be a string, got " + typeName(value)); + } + return (String) value; + } + + private static String typeName(@Nullable Object value) { + if (value == null) { + return "null"; + } + if (value instanceof List) { + return "array"; + } + if (value instanceof Map) { + return "table"; + } + if (value instanceof Number) { + return "number"; + } + if (value instanceof Boolean) { + return "boolean"; + } + if (value instanceof String) { + return "string"; + } + return value.getClass().getSimpleName(); + } + + private static void validate(SecretCache.SecretIds ids, ClientConfigProfile profile) { + List conflicts = new ArrayList<>(); + if (!ids.apiKey.isEmpty() && !isEmpty(profile.getApiKey())) { + conflicts.add("API key secret ID cannot be set with an API key"); + } + ClientConfigTLS tls = profile.getTls(); + if (tls != null) { + if (!ids.clientCert.isEmpty() + && (!isEmpty(tls.getClientCertPath()) || !isEmpty(tls.getClientCertData()))) { + conflicts.add("TLS client cert secret ID cannot be set with client cert path or data"); + } + if (!ids.clientKey.isEmpty() + && (!isEmpty(tls.getClientKeyPath()) || !isEmpty(tls.getClientKeyData()))) { + conflicts.add("TLS client key secret ID cannot be set with client key path or data"); + } + if (!ids.serverCaCert.isEmpty() + && (!isEmpty(tls.getServerCACertPath()) || !isEmpty(tls.getServerCACertData()))) { + conflicts.add( + "TLS server CA cert secret ID cannot be set with server CA cert path or data"); + } + if (ids.hasTls() && Boolean.TRUE.equals(tls.isDisabled())) { + conflicts.add("TLS secret IDs cannot be set when TLS is disabled"); + } + } + if (ids.clientCert.isEmpty() != ids.clientKey.isEmpty()) { + conflicts.add("TLS client cert and key secret IDs must be set together"); + } + if (!conflicts.isEmpty()) { + Collections.sort(conflicts); + throw new IllegalArgumentException("invalid config: " + String.join("; ", conflicts)); + } + } + + private static boolean isEmpty(@Nullable String s) { + return s == null || s.isEmpty(); + } + + private static boolean isEmpty(@Nullable byte[] b) { + return b == null || b.length == 0; + } + + /** Builder for {@link SecretsManagerExtension}. */ + public static final class Builder { + @Nullable private Object secretsClient; + @Nullable private Duration refreshInterval; + @Nullable private Duration maxStaleness; + private Map processEnv = System.getenv(); + + private Builder() {} + + /** + * Sets the Secrets Manager client used to fetch secrets. If unset, secrets are fetched through + * the AWS Parameters and Secrets Lambda Extension if it is detected, and otherwise with a + * client created from the default AWS config, which requires {@code + * software.amazon.awssdk:secretsmanager} on the classpath. + */ + public Builder setSecretsClient(@Nullable SecretsManagerClient secretsClient) { + this.secretsClient = secretsClient; + return this; + } + + /** + * Sets how long a fetched secret is used before it is refreshed. If unset, defaults to the + * {@code SECRETS_MANAGER_TTL} env var (non-negative whole seconds, where 0 refreshes on every + * use) as used by the AWS Parameters and Secrets Lambda Extension, or 5 minutes if that is + * unset. + */ + public Builder setRefreshInterval(@Nullable Duration refreshInterval) { + this.refreshInterval = refreshInterval; + return this; + } + + /** + * Sets how long after its last successful fetch a secret may still be used if refreshing it + * fails. Defaults to 24 hours. Values less than the refresh interval are treated as the refresh + * interval. + */ + public Builder setMaxStaleness(@Nullable Duration maxStaleness) { + this.maxStaleness = maxStaleness; + return this; + } + + /** Overrides the process env, for tests. */ + Builder setProcessEnv(Map processEnv) { + this.processEnv = Objects.requireNonNull(processEnv); + return this; + } + + public SecretsManagerExtension build() { + return new SecretsManagerExtension(this); + } + } +} diff --git a/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/SecretsManagerPlugin.java b/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/SecretsManagerPlugin.java new file mode 100644 index 0000000000..c23b0640ae --- /dev/null +++ b/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/SecretsManagerPlugin.java @@ -0,0 +1,163 @@ +package io.temporal.aws.secretsmanager; + +import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts; +import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext; +import io.grpc.netty.shaded.io.netty.handler.ssl.SslContextBuilder; +import io.grpc.netty.shaded.io.netty.handler.ssl.util.InsecureTrustManagerFactory; +import io.temporal.authorization.AuthorizationGrpcMetadataProvider; +import io.temporal.envconfig.ClientConfigTLS; +import io.temporal.serviceclient.GrpcMetadataProvider; +import io.temporal.serviceclient.WorkflowServiceStubs; +import io.temporal.serviceclient.WorkflowServiceStubsOptions; +import io.temporal.serviceclient.WorkflowServiceStubsPlugin; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Collection; +import java.util.function.Supplier; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * Fetches the configured secrets when service stubs are created, and applies them to the stubs' + * options. + */ +final class SecretsManagerPlugin implements WorkflowServiceStubsPlugin { + static final String NAME = "io.temporal.aws.secretsmanager"; + + final SecretCache.SecretIds ids; + final SecretCache cache; + + /** + * The profile's TLS config if TLS isn't disabled, for which envconfig created an SSL context that + * this plugin replaces, or null. + */ + @Nullable private final ClientConfigTLS profileTls; + + /** + * The API key is refreshed at most this often, and retried this long after a failed refresh. + * Checking more often would only be cache hits, and with a refresh interval of 0 (no caching), + * this bounds the fetch rate. Overridable in tests. + */ + volatile long minRefreshIntervalNanos = SecretCache.FAILED_REFRESH_RETRY_INTERVAL_NANOS; + + /** + * Telemetry for the final options of the most recently created stubs. Read when used rather than + * captured, since the final options are only seen in {@link #connectServiceClient}, after {@link + * #configureServiceStubs} installs the API key supplier. + */ + private volatile Telemetry telemetry = new Telemetry(null); + + SecretsManagerPlugin( + SecretCache.SecretIds ids, SecretCache cache, @Nullable ClientConfigTLS profileTls) { + this.ids = ids; + this.cache = cache; + this.profileTls = profileTls; + } + + @Nonnull + @Override + public String getName() { + return NAME; + } + + @Override + public void configureServiceStubs(@Nonnull WorkflowServiceStubsOptions.Builder builder) { + WorkflowServiceStubsOptions current = builder.build(); + + SecretCache.CacheEntry apiKey = cache.apiKey; + if (apiKey != null) { + Collection providers = current.getGrpcMetadataProviders(); + if (providers != null) { + for (GrpcMetadataProvider provider : providers) { + if (provider instanceof AuthorizationGrpcMetadataProvider) { + throw new IllegalStateException( + "cannot use API key from AWS Secrets Manager, API key already set"); + } + } + } + // Called for each request, so a refreshed API key is used without waiting for the refresh. + // Like an API key from envconfig, this enables TLS unless it's explicitly configured. + builder.addApiKey(() -> cache.getNow(apiKey, minRefreshIntervalNanos, telemetry)); + } + + if (ids.hasTls()) { + if (profileTls == null && current.getSslContext() != null) { + throw new IllegalStateException( + "cannot use TLS config from AWS Secrets Manager, SSL context already set"); + } + // The SSL context must be set on the options, so these fetches can't wait for + // connectServiceClient and only report to a metrics scope that's already set + builder.setSslContext(sslContext(new Telemetry(current.getMetricsScope()))); + } + } + + @Nonnull + @Override + public WorkflowServiceStubs connectServiceClient( + @Nonnull WorkflowServiceStubsOptions options, @Nonnull Supplier next) { + // Unlike in configureServiceStubs, these are the final options, after all plugins have + // configured them and defaults are applied + Telemetry tel = new Telemetry(options.getMetricsScope()); + telemetry = tel; + // Fetched now so that creating the stubs fails on a bad secret instead of the first call + if (cache.apiKey != null) { + cache.get(cache.apiKey, tel); + } + return next.get(); + } + + /** + * Creates an SSL context from the secrets and the profile's other TLS settings, the same way + * envconfig does from the profile alone. + */ + private SslContext sslContext(Telemetry tel) { + SslContextBuilder builder = GrpcSslContexts.forClient(); + try { + if (cache.serverCaCert != null) { + builder.trustManager(cache.get(cache.serverCaCert, tel)); + } else { + try (InputStream ca = + profileTls != null + ? open(profileTls.getServerCACertPath(), profileTls.getServerCACertData()) + : null) { + if (ca != null) { + builder.trustManager(ca); + } else if (profileTls != null + && Boolean.TRUE.equals(profileTls.isDisableHostVerification())) { + builder.trustManager(InsecureTrustManagerFactory.INSTANCE); + } + } + } + + if (cache.clientCert != null) { + SecretCache.ClientCertPair pair = cache.get(cache.clientCert, tel); + builder.keyManager(pair.key, pair.chain); + } else if (profileTls != null) { + try (InputStream cert = + open(profileTls.getClientCertPath(), profileTls.getClientCertData()); + InputStream key = open(profileTls.getClientKeyPath(), profileTls.getClientKeyData())) { + if (cert != null && key != null) { + builder.keyManager(cert, key); + } + } + } + return builder.build(); + } catch (IOException e) { + throw new RuntimeException("Unable to create SSL context", e); + } + } + + @Nullable + private static InputStream open(@Nullable String path, @Nullable byte[] data) throws IOException { + if (path != null && !path.isEmpty()) { + return Files.newInputStream(Paths.get(path)); + } + if (data != null && data.length > 0) { + return new ByteArrayInputStream(data); + } + return null; + } +} diff --git a/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/Telemetry.java b/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/Telemetry.java new file mode 100644 index 0000000000..de05e5a94f --- /dev/null +++ b/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/Telemetry.java @@ -0,0 +1,109 @@ +package io.temporal.aws.secretsmanager; + +import com.uber.m3.tally.NoopScope; +import com.uber.m3.tally.Scope; +import java.util.HashMap; +import java.util.Map; +import javax.annotation.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Metrics for a client's metric scope. Metric and label names must be the same across SDKs. */ +class Telemetry { + private static final Logger log = LoggerFactory.getLogger(Telemetry.class); + + static final String METRIC_FETCH_FAILED = "temporal_aws_secrets_manager_fetch_failed"; + static final String METRIC_FETCH_LATENCY = "temporal_aws_secrets_manager_fetch_latency"; + static final String METRIC_CACHE_LOOKUP = "temporal_aws_secrets_manager_cache_lookup"; + + static final String LABEL_SECRET_TYPE = "secret_type"; + static final String LABEL_FETCH_PATH = "fetch_path"; + static final String LABEL_FAILURE_REASON = "failure_reason"; + static final String LABEL_FALLBACK = "fallback"; + static final String LABEL_OUTCOME = "outcome"; + static final String LABEL_RESULT = "result"; + + // secret_type values. Client cert and key secrets are fetched together and reported as + // tls_client_cert in metrics. + static final String SECRET_TYPE_API_KEY = "api_key"; + static final String SECRET_TYPE_TLS_CLIENT_CERT = "tls_client_cert"; + static final String SECRET_TYPE_TLS_CLIENT_KEY = "tls_client_key"; + static final String SECRET_TYPE_TLS_SERVER_CA_CERT = "tls_server_ca_cert"; + + // fetch_path values: the Lambda extension, or a direct fetch with the AWS SDK + static final String FETCH_PATH_EXTENSION = "extension"; + static final String FETCH_PATH_SDK = "sdk"; + + // AWS error codes mapped to failure_reason values. Other codes are "other". + private static final Map AWS_ERROR_CODE_FAILURE_REASONS = new HashMap<>(); + + static { + AWS_ERROR_CODE_FAILURE_REASONS.put("ResourceNotFoundException", "resource_not_found"); + AWS_ERROR_CODE_FAILURE_REASONS.put("AccessDeniedException", "access_denied"); + AWS_ERROR_CODE_FAILURE_REASONS.put("DecryptionFailure", "decryption_failure"); + AWS_ERROR_CODE_FAILURE_REASONS.put("InvalidRequestException", "invalid_request"); + AWS_ERROR_CODE_FAILURE_REASONS.put("InvalidParameterException", "invalid_request"); + AWS_ERROR_CODE_FAILURE_REASONS.put("ThrottlingException", "throttling"); + AWS_ERROR_CODE_FAILURE_REASONS.put("InternalServiceError", "internal_service_error"); + } + + private final Scope scope; + + Telemetry(@Nullable Scope scope) { + this.scope = scope != null ? scope : new NoopScope(); + } + + /** Returns the failure_reason metric label for a fetch error. */ + static String classifyFailure(Throwable err) { + if (err instanceof FetchErrors.InvalidResponseException) { + return "invalid_response"; + } + if (err instanceof FetchErrors.LambdaExtensionStatusException) { + int status = ((FetchErrors.LambdaExtensionStatusException) err).status; + if (status >= 400 && status < 500) { + return "client_error"; + } + if (status >= 500 && status < 600) { + return "server_error"; + } + return "other"; + } + if (err instanceof FetchErrors.FetchTimeoutException) { + return "timeout"; + } + if (err instanceof FetchErrors.AwsErrorException) { + String code = ((FetchErrors.AwsErrorException) err).errorCode; + String reason = code != null ? AWS_ERROR_CODE_FAILURE_REASONS.get(code) : null; + return reason != null ? reason : "other"; + } + // The Lambda extension doesn't return the underlying AWS error, only an HTTP status with a + // generic message, so its errors are only classified by status class + return "other"; + } + + void cacheLookup(Map tags) { + record(() -> scope.tagged(tags).counter(METRIC_CACHE_LOOKUP).inc(1)); + } + + void fetchFailed(Map tags) { + record(() -> scope.tagged(tags).counter(METRIC_FETCH_FAILED).inc(1)); + } + + void fetchLatency(Map tags, long nanos) { + record( + () -> + scope + .tagged(tags) + .timer(METRIC_FETCH_LATENCY) + .record(com.uber.m3.util.Duration.ofNanos(nanos))); + } + + /** Records a metric, never throwing, since recording failures shouldn't affect fetches. */ + private static void record(Runnable fn) { + try { + fn.run(); + } catch (RuntimeException e) { + log.warn("Failed recording AWS Secrets Manager metric", e); + } + } +} diff --git a/contrib/temporal-aws-secrets-manager/src/test/java/io/temporal/aws/secretsmanager/FetcherTest.java b/contrib/temporal-aws-secrets-manager/src/test/java/io/temporal/aws/secretsmanager/FetcherTest.java new file mode 100644 index 0000000000..e29765cd01 --- /dev/null +++ b/contrib/temporal-aws-secrets-manager/src/test/java/io/temporal/aws/secretsmanager/FetcherTest.java @@ -0,0 +1,249 @@ +package io.temporal.aws.secretsmanager; + +import static org.junit.Assert.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.sun.net.httpserver.HttpServer; +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import software.amazon.awssdk.awscore.exception.AwsErrorDetails; +import software.amazon.awssdk.core.SdkBytes; +import software.amazon.awssdk.core.exception.ApiCallTimeoutException; +import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient; +import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueRequest; +import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueResponse; +import software.amazon.awssdk.services.secretsmanager.model.SecretsManagerException; + +public class FetcherTest { + private HttpServer server; + private int port; + private final List requests = new ArrayList<>(); + private volatile Function respond = id -> new String[] {"404", "not found"}; + + @Before + public void setUp() throws Exception { + server = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0); + server.createContext( + "/", + exchange -> { + String query = exchange.getRequestURI().getRawQuery(); + synchronized (requests) { + requests.add( + exchange.getRequestURI().getRawPath() + + "?" + + query + + " token=" + + exchange.getRequestHeaders().getFirst("X-Aws-Parameters-Secrets-Token")); + } + String id = java.net.URLDecoder.decode(query.substring("secretId=".length()), "UTF-8"); + String[] resp = respond.apply(id); + byte[] body = resp[1].getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(Integer.parseInt(resp[0]), body.length); + try (OutputStream out = exchange.getResponseBody()) { + out.write(body); + } + }); + server.start(); + port = server.getAddress().getPort(); + } + + @After + public void tearDown() { + server.stop(0); + } + + private LambdaExtensionFetcher extensionFetcher() { + return new LambdaExtensionFetcher( + String.valueOf(port), Collections.singletonMap("AWS_SESSION_TOKEN", "token")); + } + + @Test + public void lambdaExtensionFetcher() throws Exception { + respond = + id -> + id.equals("binary/secret") + ? new String[] { + "200", + "{\"SecretBinary\": \"" + + Base64.getEncoder().encodeToString(new byte[] {1, 2, 3}) + + "\", \"VersionId\": \"v2\"}" + } + : new String[] {"200", "{\"SecretString\": \"value\", \"VersionId\": \"v1\"}"}; + LambdaExtensionFetcher fetcher = extensionFetcher(); + assertEquals("extension", fetcher.path()); + Fetcher.SecretValue string = fetcher.fetch("string"); + assertEquals("value", new String(string.value, StandardCharsets.UTF_8)); + assertEquals("v1", string.versionId); + Fetcher.SecretValue binary = fetcher.fetch("binary/secret"); + assertArrayEquals(new byte[] {1, 2, 3}, binary.value); + assertEquals("v2", binary.versionId); + assertEquals( + new ArrayList<>( + java.util.Arrays.asList( + "/secretsmanager/get?secretId=string token=token", + "/secretsmanager/get?secretId=binary%2Fsecret token=token")), + requests); + } + + @Test + public void lambdaExtensionFetcherErrors() { + Object[][] cases = { + {"400", "bad", "client_error"}, + {"500", "oops", "server_error"}, + {"200", "not json", "invalid_response"}, + {"200", "\"string\"", "invalid_response"}, + {"200", "{\"SecretString\": 1}", "invalid_response"}, + {"200", "{\"VersionId\": \"v1\"}", "invalid_response"}, + }; + for (Object[] c : cases) { + respond = id -> new String[] {(String) c[0], (String) c[1]}; + Exception e = assertThrows(Exception.class, () -> extensionFetcher().fetch("id")); + assertEquals(c[1] + ": " + e, c[2], Telemetry.classifyFailure(e)); + } + } + + @Test + public void lambdaExtensionFetcherConnectionRefused() throws Exception { + int closedPort; + try (ServerSocket socket = new ServerSocket(0, 0, InetAddress.getLoopbackAddress())) { + closedPort = socket.getLocalPort(); + } + Exception e = + assertThrows( + Exception.class, + () -> + new LambdaExtensionFetcher(String.valueOf(closedPort), Collections.emptyMap()) + .fetch("id")); + assertEquals("other", Telemetry.classifyFailure(e)); + } + + @Test + public void sdkFetcher() throws Exception { + SecretsManagerClient client = mock(SecretsManagerClient.class); + List reqs = new ArrayList<>(); + when(client.getSecretValue(any(GetSecretValueRequest.class))) + .thenAnswer( + inv -> { + GetSecretValueRequest req = inv.getArgument(0); + reqs.add(req); + return req.secretId().equals("binary") + ? GetSecretValueResponse.builder() + .secretBinary(SdkBytes.fromByteArray(new byte[] {1, 2})) + .versionId("v2") + .build() + : GetSecretValueResponse.builder().secretString("value").versionId("v1").build(); + }); + Fetcher fetcher = SdkFetcher.create(client); + assertEquals("sdk", fetcher.path()); + assertEquals("value", new String(fetcher.fetch("string").value, StandardCharsets.UTF_8)); + assertArrayEquals(new byte[] {1, 2}, fetcher.fetch("binary").value); + // Each request is bounded by the fetch timeout + assertTrue(reqs.get(0).overrideConfiguration().get().apiCallTimeout().isPresent()); + } + + @Test + public void sdkFetcherErrors() { + SecretsManagerClient client = mock(SecretsManagerClient.class); + when(client.getSecretValue(any(GetSecretValueRequest.class))) + .thenThrow( + SecretsManagerException.builder() + .message("not found") + .awsErrorDetails( + AwsErrorDetails.builder().errorCode("ResourceNotFoundException").build()) + .build()) + .thenThrow(ApiCallTimeoutException.create(10_000)) + .thenReturn(GetSecretValueResponse.builder().versionId("v1").build()); + Fetcher fetcher = SdkFetcher.create(client); + assertEquals( + "resource_not_found", + Telemetry.classifyFailure(assertThrows(Exception.class, () -> fetcher.fetch("id")))); + assertEquals( + "timeout", + Telemetry.classifyFailure(assertThrows(Exception.class, () -> fetcher.fetch("id")))); + Exception e = assertThrows(Exception.class, () -> fetcher.fetch("id")); + assertEquals("invalid_response", Telemetry.classifyFailure(e)); + assertEquals("secret has no value", e.getMessage()); + } + + @Test + public void classifyAwsErrors() { + Map cases = new HashMap<>(); + cases.put("ResourceNotFoundException", "resource_not_found"); + cases.put("AccessDeniedException", "access_denied"); + cases.put("DecryptionFailure", "decryption_failure"); + cases.put("InvalidRequestException", "invalid_request"); + cases.put("InvalidParameterException", "invalid_request"); + cases.put("ThrottlingException", "throttling"); + cases.put("InternalServiceError", "internal_service_error"); + cases.put("SomethingElse", "other"); + for (Map.Entry c : cases.entrySet()) { + assertEquals( + c.getKey(), + c.getValue(), + Telemetry.classifyFailure( + new FetchErrors.AwsErrorException(c.getKey(), new Exception()))); + } + assertEquals( + "other", + Telemetry.classifyFailure(new FetchErrors.AwsErrorException(null, new Exception()))); + } + + @Test + public void classifyOtherErrors() { + assertEquals( + "invalid_response", + Telemetry.classifyFailure(new FetchErrors.InvalidResponseException("bad"))); + assertEquals( + "timeout", Telemetry.classifyFailure(new FetchErrors.FetchTimeoutException("slow", null))); + assertEquals( + "client_error", + Telemetry.classifyFailure(new FetchErrors.LambdaExtensionStatusException(403, ""))); + assertEquals( + "server_error", + Telemetry.classifyFailure(new FetchErrors.LambdaExtensionStatusException(503, ""))); + assertEquals( + "other", + Telemetry.classifyFailure(new FetchErrors.LambdaExtensionStatusException(302, ""))); + assertEquals("other", Telemetry.classifyFailure(new RuntimeException())); + } + + @Test + public void defaultFetcherSelection() { + Map env = new HashMap<>(); + env.put(Fetcher.ENV_LAMBDA_EXTENSION_PORT, "1234"); + assertTrue(Fetcher.newDefaultFetcher(env) instanceof LambdaExtensionFetcher); + + // Empty disables detection, even in Lambda + env.put(Fetcher.ENV_LAMBDA_EXTENSION_PORT, ""); + env.put(Fetcher.ENV_LAMBDA_FUNCTION_NAME, "fn"); + assertTrue(Fetcher.newDefaultFetcher(env) instanceof SdkFetcher); + + // Not probed outside Lambda + assertTrue(Fetcher.newDefaultFetcher(new HashMap<>()) instanceof SdkFetcher); + } + + @Test + public void probe() throws Exception { + assertTrue(Fetcher.probe(InetAddress.getLoopbackAddress().getHostAddress(), port)); + int closedPort; + try (ServerSocket socket = new ServerSocket(0, 0, InetAddress.getLoopbackAddress())) { + closedPort = socket.getLocalPort(); + } + assertFalse(Fetcher.probe(InetAddress.getLoopbackAddress().getHostAddress(), closedPort)); + } +} diff --git a/contrib/temporal-aws-secrets-manager/src/test/java/io/temporal/aws/secretsmanager/SecretCacheTest.java b/contrib/temporal-aws-secrets-manager/src/test/java/io/temporal/aws/secretsmanager/SecretCacheTest.java new file mode 100644 index 0000000000..944556bf31 --- /dev/null +++ b/contrib/temporal-aws-secrets-manager/src/test/java/io/temporal/aws/secretsmanager/SecretCacheTest.java @@ -0,0 +1,396 @@ +package io.temporal.aws.secretsmanager; + +import static io.temporal.aws.secretsmanager.TestHelpers.*; +import static org.junit.Assert.*; + +import io.temporal.aws.secretsmanager.SecretCache.ClientCertPair; +import io.temporal.aws.secretsmanager.SecretCache.SecretFetchException; +import io.temporal.aws.secretsmanager.SecretCache.SecretIds; +import java.security.cert.X509Certificate; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; +import org.junit.Test; + +public class SecretCacheTest { + private static final long REFRESH = TimeUnit.MINUTES.toNanos(1); + private static final long MAX_STALENESS = TimeUnit.HOURS.toNanos(1); + private static final long RETRY = SecretCache.FAILED_REFRESH_RETRY_INTERVAL_NANOS; + + private final FakeFetcher fetcher = new FakeFetcher(); + private final AtomicLong clock = new AtomicLong(1_000_000_000L); + private final RecordingTelemetry tel = new RecordingTelemetry(); + + private SecretCache newCache(SecretIds ids, long refresh) { + SecretCache cache = new SecretCache(fetcher, ids, refresh, MAX_STALENESS); + cache.now = clock::get; + return cache; + } + + private SecretCache newApiKeyCache(long refresh) { + return newCache(new SecretIds("key", "", "", ""), refresh); + } + + private static Map tags(String... kv) { + Map m = new HashMap<>(); + for (int i = 0; i < kv.length; i += 2) { + m.put(kv[i], kv[i + 1]); + } + return m; + } + + private List results() { + return tel.take(Telemetry.METRIC_CACHE_LOOKUP).stream() + .map(t -> t.get(Telemetry.LABEL_RESULT)) + .collect(Collectors.toList()); + } + + @Test + public void hitMissAndRotation() { + fetcher.set("key", "v1", "ver1"); + SecretCache cache = newApiKeyCache(REFRESH); + assertEquals("v1", cache.get(cache.apiKey, tel)); + assertEquals("v1", cache.get(cache.apiKey, tel)); + assertEquals(1, fetcher.calls("key")); + assertEquals( + Arrays.asList( + tags("secret_type", "api_key", "fetch_path", "sdk", "result", "miss"), + tags("secret_type", "api_key", "fetch_path", "sdk", "result", "hit")), + tel.take(Telemetry.METRIC_CACHE_LOOKUP)); + assertEquals( + Arrays.asList(tags("secret_type", "api_key", "fetch_path", "sdk", "outcome", "success")), + tel.take(Telemetry.METRIC_FETCH_LATENCY)); + + clock.addAndGet(REFRESH); + fetcher.set("key", "v2", "ver2"); + assertEquals("v2", cache.get(cache.apiKey, tel)); + assertEquals(2, fetcher.calls("key")); + } + + @Test + public void zeroRefreshIntervalAlwaysFetches() { + fetcher.set("key", "v1", "ver1"); + SecretCache cache = newApiKeyCache(0); + cache.get(cache.apiKey, tel); + cache.get(cache.apiKey, tel); + assertEquals(2, fetcher.calls("key")); + } + + @Test + public void failureFallsBackToStaleValueAndWaitsBeforeRetrying() { + fetcher.set("key", "v1", "ver1"); + SecretCache cache = newApiKeyCache(REFRESH); + cache.get(cache.apiKey, tel); + tel.take(Telemetry.METRIC_CACHE_LOOKUP); + + fetcher.fail( + "key", new FetchErrors.AwsErrorException("AccessDeniedException", new Exception())); + clock.addAndGet(REFRESH); + assertEquals("v1", cache.get(cache.apiKey, tel)); + assertEquals( + Arrays.asList( + tags( + "secret_type", + "api_key", + "fetch_path", + "sdk", + "failure_reason", + "access_denied", + "fallback", + "stale")), + tel.take(Telemetry.METRIC_FETCH_FAILED)); + + // Not retried until the retry interval has passed + clock.addAndGet(RETRY - 1); + assertEquals("v1", cache.get(cache.apiKey, tel)); + assertEquals(2, fetcher.calls("key")); + clock.addAndGet(1); + assertEquals("v1", cache.get(cache.apiKey, tel)); + assertEquals(3, fetcher.calls("key")); + assertEquals( + Arrays.asList("miss_with_fallback", "miss_with_fallback", "miss_with_fallback"), results()); + + fetcher.set("key", "v1", "ver1"); + clock.addAndGet(RETRY); + assertEquals("v1", cache.get(cache.apiKey, tel)); + assertNull(cache.apiKey.failedAt); + } + + @Test + public void failureWithoutPreviousValueThrows() { + fetcher.fail( + "key", + new FetchErrors.AwsErrorException("ResourceNotFoundException", new Exception("nope"))); + SecretCache cache = newApiKeyCache(REFRESH); + SecretFetchException e = + assertThrows(SecretFetchException.class, () -> cache.get(cache.apiKey, tel)); + assertEquals("failed fetching AWS Secrets Manager secret \"key\": nope", e.getMessage()); + assertEquals("resource_not_found", e.reason); + assertEquals( + "none", tel.take(Telemetry.METRIC_FETCH_FAILED).get(0).get(Telemetry.LABEL_FALLBACK)); + assertTrue(results().isEmpty()); + } + + @Test + public void staleValueIsNotUsedPastMaxStaleness() { + fetcher.set("key", "v1", "ver1"); + SecretCache cache = newApiKeyCache(REFRESH); + cache.get(cache.apiKey, tel); + fetcher.fail("key", new Exception("down")); + clock.addAndGet(MAX_STALENESS); + assertThrows(SecretFetchException.class, () -> cache.get(cache.apiKey, tel)); + } + + @Test + public void concurrentGetsShareAFetch() throws Exception { + fetcher.set("key", "v1", "ver1"); + fetcher.gate = new CountDownLatch(1); + SecretCache cache = newApiKeyCache(REFRESH); + ExecutorService pool = Executors.newFixedThreadPool(2); + try { + Future a = pool.submit(() -> cache.get(cache.apiKey, tel)); + Future b = pool.submit(() -> cache.get(cache.apiKey, tel)); + waitUntil(() -> fetcher.calls("key") == 1); + fetcher.gate.countDown(); + assertEquals("v1", a.get(5, TimeUnit.SECONDS)); + assertEquals("v1", b.get(5, TimeUnit.SECONDS)); + assertEquals(1, fetcher.calls("key")); + } finally { + pool.shutdownNow(); + } + } + + @Test + public void refreshingServesStaleValueToOtherCallers() throws Exception { + fetcher.set("key", "v1", "ver1"); + SecretCache cache = newApiKeyCache(REFRESH); + cache.get(cache.apiKey, tel); + + fetcher.set("key", "v2", "ver2"); + fetcher.gate = new CountDownLatch(1); + clock.addAndGet(REFRESH); + ExecutorService pool = Executors.newSingleThreadExecutor(); + try { + Future refreshing = pool.submit(() -> cache.get(cache.apiKey, tel)); + waitUntil(() -> fetcher.calls("key") == 2); + assertEquals("v1", cache.get(cache.apiKey, tel)); + fetcher.gate.countDown(); + assertEquals("v2", refreshing.get(5, TimeUnit.SECONDS)); + } finally { + pool.shutdownNow(); + } + } + + @Test + public void callerTimingOutDoesNotCancelFetch() throws Exception { + fetcher.set("key", "v1", "ver1"); + fetcher.gate = new CountDownLatch(1); + SecretCache cache = newApiKeyCache(REFRESH); + cache.fetchTimeoutNanos = TimeUnit.MILLISECONDS.toNanos(10); + SecretCache.FetchWaitTimeoutException e = + assertThrows( + SecretCache.FetchWaitTimeoutException.class, () -> cache.get(cache.apiKey, tel)); + assertEquals( + "timed out after 10ms waiting for AWS Secrets Manager secret fetch", e.getMessage()); + + fetcher.gate.countDown(); + waitUntil(() -> inFlight(cache) == null); + assertEquals("v1", cache.get(cache.apiKey, tel)); + assertEquals(1, fetcher.calls("key")); + } + + @Test + public void timedOutWaitFallsBackToStaleValue() throws Exception { + fetcher.set("key", "v1", "ver1"); + SecretCache cache = newApiKeyCache(0); + cache.get(cache.apiKey, tel); + tel.take(Telemetry.METRIC_CACHE_LOOKUP); + + fetcher.gate = new CountDownLatch(1); + cache.fetchTimeoutNanos = TimeUnit.MILLISECONDS.toNanos(10); + clock.addAndGet(1); + assertEquals("v1", cache.get(cache.apiKey, tel)); + assertEquals(Arrays.asList("miss_with_fallback"), results()); + fetcher.gate.countDown(); + waitUntil(() -> inFlight(cache) == null); + } + + @Test + public void failedFetchStartDoesNotWedgeCache() { + fetcher.set("key", "v1", "ver1"); + SecretCache cache = newApiKeyCache(REFRESH); + cache.executor = + r -> { + throw new RejectedExecutionException("no threads"); + }; + SecretFetchException e = + assertThrows(SecretFetchException.class, () -> cache.get(cache.apiKey, tel)); + assertEquals("failed starting AWS Secrets Manager secret fetch", e.getMessage()); + assertNull(inFlight(cache)); + + cache.executor = Runnable::run; + assertEquals("v1", cache.get(cache.apiKey, tel)); + } + + @Test + public void getNowRefreshesInBackgroundWhenDue() throws Exception { + fetcher.set("key", "v1", "ver1"); + SecretCache cache = newApiKeyCache(0); + cache.get(cache.apiKey, tel); + long min = TimeUnit.SECONDS.toNanos(1); + + // A zero refresh interval is bounded by the minimum + fetcher.set("key", "v2", "ver2"); + assertEquals("v1", cache.getNow(cache.apiKey, min, tel)); + assertNull(inFlight(cache)); + clock.addAndGet(min); + assertEquals("v1", cache.getNow(cache.apiKey, min, tel)); + waitUntil(() -> "v2".equals(cache.getNow(cache.apiKey, min, tel))); + assertEquals(2, fetcher.calls("key")); + + // A failed refresh keeps the current value, even past max staleness, and isn't retried until + // the retry interval has passed + fetcher.fail("key", new Exception("down")); + clock.addAndGet(MAX_STALENESS); + assertEquals("v2", cache.getNow(cache.apiKey, min, tel)); + waitUntil(() -> cache.apiKey.failedAt != null && inFlight(cache) == null); + assertEquals("v2", cache.getNow(cache.apiKey, min, tel)); + assertEquals(3, fetcher.calls("key")); + clock.addAndGet(RETRY); + cache.getNow(cache.apiKey, min, tel); + waitUntil(() -> fetcher.calls("key") == 4 && inFlight(cache) == null); + } + + @Test + public void bundledClientCertAndKey() { + fetcher.set("bundle", "junk\n" + TEST_CERT_A + TEST_KEY_A, "ver1"); + SecretCache cache = newCache(new SecretIds("", "bundle", "bundle", ""), REFRESH); + ClientCertPair pair = cache.get(cache.clientCert, tel); + assertEquals(1, pair.chain.length); + assertEquals( + "CN=aws-secrets-manager-test-a", pair.chain[0].getSubjectX500Principal().getName()); + assertEquals("EC", pair.key.getAlgorithm()); + assertEquals(1, fetcher.calls("bundle")); + } + + @Test + public void separateClientCertAndKey() { + fetcher.set("cert", TEST_CERT_A, "ver1"); + fetcher.set("key", TEST_KEY_A, "ver2"); + SecretCache cache = newCache(new SecretIds("", "cert", "key", ""), REFRESH); + cache.get(cache.clientCert, tel); + // Both are reported as the client cert in metrics + assertEquals( + Arrays.asList( + tags("secret_type", "tls_client_cert", "fetch_path", "sdk", "result", "miss")), + tel.take(Telemetry.METRIC_CACHE_LOOKUP)); + } + + @Test + public void mismatchedClientCertAndKeyKeepThePreviousPair() { + fetcher.set("cert", TEST_CERT_A, "ver1"); + fetcher.set("key", TEST_KEY_A, "ver1"); + SecretCache cache = newCache(new SecretIds("", "cert", "key", ""), REFRESH); + cache.get(cache.clientCert, tel); + + // Fetched partway through rotating the separate secrets + fetcher.set("cert", TEST_CERT_B, "ver2"); + clock.addAndGet(REFRESH); + assertEquals( + subject(cache.get(cache.clientCert, tel).chain[0]), "CN=aws-secrets-manager-test-a"); + assertEquals( + "invalid_response", + tel.take(Telemetry.METRIC_FETCH_FAILED).get(0).get(Telemetry.LABEL_FAILURE_REASON)); + + fetcher.set("key", TEST_KEY_B, "ver2"); + clock.addAndGet(RETRY); + assertEquals( + subject(cache.get(cache.clientCert, tel).chain[0]), "CN=aws-secrets-manager-test-b"); + } + + @Test + public void invalidClientCertAndKey() { + String[][] cases = { + {TEST_KEY_A, TEST_KEY_A, "no PEM certificate found in TLS client cert secret \"cert\""}, + {TEST_CERT_A, TEST_CERT_A, "no PEM private key found in TLS client key secret \"key\""}, + { + TEST_CERT_A, + TEST_KEY_B, + "TLS client cert and key from AWS Secrets Manager secrets \"cert\" and \"key\" don't match" + }, + { + TEST_CERT_A, + TEST_KEY_A_EC, + "TLS client key secret \"key\" is not a PKCS#8 key: its PEM header is \"-----BEGIN EC" + + " PRIVATE KEY-----\" instead of \"-----BEGIN PRIVATE KEY-----\". Convert it with:" + + " openssl pkcs8 -topk8 -nocrypt -in key.pem -out key-pkcs8.pem" + }, + { + "-----BEGIN CERTIFICATE-----\nbm9wZQ==\n-----END CERTIFICATE-----", + TEST_KEY_A, + null // Message from the JDK + }, + { + TEST_CERT_A, + "-----BEGIN PRIVATE KEY-----\nbm9wZQ==\n-----END PRIVATE KEY-----", + "invalid PEM private key in TLS client key secret \"key\"" + }, + }; + for (String[] c : cases) { + fetcher.set("cert", c[0], "ver1"); + fetcher.set("key", c[1], "ver1"); + SecretCache cache = newCache(new SecretIds("", "cert", "key", ""), REFRESH); + SecretFetchException e = + assertThrows(SecretFetchException.class, () -> cache.get(cache.clientCert, tel)); + if (c[2] != null) { + assertEquals(c[2], e.getMessage()); + } else { + assertTrue(e.getMessage(), e.getMessage().startsWith("invalid PEM certificate in")); + } + assertEquals("invalid_response", e.reason); + } + } + + @Test + public void invalidUtf8ApiKey() { + fetcher.set("key", new byte[] {(byte) 0xff, (byte) 0xfe}, "ver1"); + SecretCache cache = newApiKeyCache(REFRESH); + SecretFetchException e = + assertThrows(SecretFetchException.class, () -> cache.get(cache.apiKey, tel)); + assertEquals("API key secret is not valid UTF-8", e.getMessage()); + assertEquals("invalid_response", e.reason); + } + + @Test + public void serverCaCert() { + fetcher.set("ca", TEST_CERT_A + "\n" + TEST_CERT_B, "ver1"); + SecretCache cache = newCache(new SecretIds("", "", "", "ca"), REFRESH); + X509Certificate[] certs = cache.get(cache.serverCaCert, tel); + assertEquals(2, certs.length); + + fetcher.set("ca", "nothing here", "ver1"); + SecretCache cache2 = newCache(new SecretIds("", "", "", "ca"), REFRESH); + SecretFetchException e = + assertThrows(SecretFetchException.class, () -> cache2.get(cache2.serverCaCert, tel)); + assertEquals("no PEM certificates found in TLS server CA cert secret \"ca\"", e.getMessage()); + } + + private static String subject(X509Certificate cert) { + return cert.getSubjectX500Principal().getName(); + } + + private static Object inFlight(SecretCache cache) { + synchronized (cache.apiKey != null ? cache.apiKey : cache.clientCert) { + return cache.apiKey != null ? cache.apiKey.inFlight : cache.clientCert.inFlight; + } + } +} diff --git a/contrib/temporal-aws-secrets-manager/src/test/java/io/temporal/aws/secretsmanager/SecretsManagerExtensionTest.java b/contrib/temporal-aws-secrets-manager/src/test/java/io/temporal/aws/secretsmanager/SecretsManagerExtensionTest.java new file mode 100644 index 0000000000..b861373d14 --- /dev/null +++ b/contrib/temporal-aws-secrets-manager/src/test/java/io/temporal/aws/secretsmanager/SecretsManagerExtensionTest.java @@ -0,0 +1,499 @@ +package io.temporal.aws.secretsmanager; + +import static io.temporal.aws.secretsmanager.TestHelpers.*; +import static org.junit.Assert.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.RETURNS_DEEP_STUBS; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.uber.m3.tally.Scope; +import io.grpc.Metadata; +import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts; +import io.temporal.authorization.AuthorizationGrpcMetadataProvider; +import io.temporal.envconfig.ClientConfigProfile; +import io.temporal.envconfig.LoadClientConfigProfileOptions; +import io.temporal.serviceclient.GrpcMetadataProvider; +import io.temporal.serviceclient.WorkflowServiceStubs; +import io.temporal.serviceclient.WorkflowServiceStubsOptions; +import io.temporal.serviceclient.WorkflowServiceStubsPlugin; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import org.junit.Test; +import software.amazon.awssdk.awscore.exception.AwsErrorDetails; +import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient; +import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueRequest; +import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueResponse; +import software.amazon.awssdk.services.secretsmanager.model.SecretsManagerException; + +public class SecretsManagerExtensionTest { + private static final String PROFILE = "[profile.default]\naddress = \"localhost:7233\"\n"; + private static final String EXT = "[profile.default.ext.aws_secrets_manager]\n"; + private static final String EXT_TLS = "[profile.default.ext.aws_secrets_manager.tls]\n"; + private static final String API_KEY_PROFILE = PROFILE + EXT + "api_key_id = \"id\"\n"; + private static final String ALL_PROFILE = + PROFILE + + EXT + + "api_key_id = \"api-key\"\n" + + EXT_TLS + + "client_cert_id = \"bundle\"\n" + + "client_key_id = \"bundle\"\n" + + "server_ca_cert_id = \"ca\"\n"; + + /** A fake Secrets Manager client that returns {@code secrets}, counting fetches of each. */ + private static final class FakeSecrets { + final Map secrets = new ConcurrentHashMap<>(); + final Map calls = new ConcurrentHashMap<>(); + private final AtomicInteger version = new AtomicInteger(); + final SecretsManagerClient client = mock(SecretsManagerClient.class); + + FakeSecrets() { + when(client.getSecretValue(any(GetSecretValueRequest.class))) + .thenAnswer( + inv -> { + String id = inv.getArgument(0).secretId(); + calls.computeIfAbsent(id, k -> new AtomicInteger()).incrementAndGet(); + Object value = secrets.get(id); + if (value == null) { + throw SecretsManagerException.builder() + .message("no secret " + id) + .awsErrorDetails( + AwsErrorDetails.builder().errorCode("ResourceNotFoundException").build()) + .build(); + } + if (value instanceof RuntimeException) { + throw (RuntimeException) value; + } + return GetSecretValueResponse.builder() + .secretString((String) value) + .versionId("v" + version.incrementAndGet()) + .build(); + }); + } + + FakeSecrets put(String id, Object value) { + secrets.put(id, value); + return this; + } + + int calls(String id) { + AtomicInteger n = calls.get(id); + return n != null ? n.get() : 0; + } + } + + private static FakeSecrets allSecrets() { + return new FakeSecrets() + .put("api-key", "secret-key") + .put("bundle", TEST_CERT_A + TEST_KEY_A) + .put("ca", TEST_CERT_A); + } + + private static ClientConfigProfile load( + String toml, Map env, SecretsManagerExtension.Builder extension) + throws Exception { + return ClientConfigProfile.load( + LoadClientConfigProfileOptions.newBuilder() + .setConfigFileData(toml.getBytes(StandardCharsets.UTF_8)) + .setEnvOverrides(env) + .setExtensions(Collections.singletonList(extension.build())) + .build()); + } + + private static SecretsManagerExtension.Builder extension(FakeSecrets secrets) { + return SecretsManagerExtension.newBuilder() + .setSecretsClient(secrets.client) + .setProcessEnv(Collections.emptyMap()); + } + + private static SecretsManagerPlugin plugin(ClientConfigProfile profile) { + assertEquals(1, profile.getPlugins().size()); + return (SecretsManagerPlugin) profile.getPlugins().get(0); + } + + private static SecretsManagerPlugin load(String toml, FakeSecrets secrets) throws Exception { + return plugin(load(toml, Collections.emptyMap(), extension(secrets))); + } + + private static String ids(SecretsManagerPlugin plugin) { + SecretCache.SecretIds ids = plugin.ids; + return String.join(",", ids.apiKey, ids.clientCert, ids.clientKey, ids.serverCaCert); + } + + /** Applies the plugin to the profile's service stubs options, without creating stubs. */ + private static WorkflowServiceStubsOptions configure(ClientConfigProfile profile) { + WorkflowServiceStubsOptions.Builder builder = + WorkflowServiceStubsOptions.newBuilder(profile.toWorkflowServiceStubsOptions()); + plugin(profile).configureServiceStubs(builder); + WorkflowServiceStubsOptions options = builder.build(); + plugin(profile).connectServiceClient(options, () -> null); + return options; + } + + private static String authorization(WorkflowServiceStubsOptions options) { + List providers = + options.getGrpcMetadataProviders().stream() + .filter(p -> p instanceof AuthorizationGrpcMetadataProvider) + .collect(Collectors.toList()); + assertEquals(1, providers.size()); + return providers + .get(0) + .getMetadata() + .get(Metadata.Key.of("authorization", Metadata.ASCII_STRING_MARSHALLER)); + } + + @Test + public void noSecretIdsMeansNoPlugin() throws Exception { + assertTrue( + load(PROFILE, Collections.emptyMap(), extension(new FakeSecrets())).getPlugins().isEmpty()); + assertTrue( + load(PROFILE + EXT, Collections.emptyMap(), extension(new FakeSecrets())) + .getPlugins() + .isEmpty()); + } + + @Test + public void secretIdsAreLoadedFromTomlAndEnv() throws Exception { + String toml = + PROFILE + + EXT + + "api_key_id = \"toml-api-key\"\n" + + EXT_TLS + + "client_cert_id = \"toml-cert\"\n" + + "client_key_id = \"toml-key\"\n" + + "server_ca_cert_id = \"toml-ca\"\n"; + assertEquals("toml-api-key,toml-cert,toml-key,toml-ca", ids(load(toml, new FakeSecrets()))); + + // Env vars override the file, and set-but-empty env vars unset it + Map env = new HashMap<>(); + env.put("TEMPORAL_EXT_AWS_SECRETS_MANAGER_API_KEY_ID", "env-api-key"); + env.put("TEMPORAL_EXT_AWS_SECRETS_MANAGER_TLS_SERVER_CA_CERT_ID", ""); + assertEquals( + "env-api-key,toml-cert,toml-key,", + ids(plugin(load(toml, env, extension(new FakeSecrets()))))); + + // Env vars are ignored when env loading is disabled + ClientConfigProfile profile = + ClientConfigProfile.load( + LoadClientConfigProfileOptions.newBuilder() + .setConfigFileData(PROFILE.getBytes(StandardCharsets.UTF_8)) + .setEnvOverrides( + Collections.singletonMap("TEMPORAL_EXT_AWS_SECRETS_MANAGER_API_KEY_ID", "id")) + .setDisableEnv(true) + .setExtensions(Collections.singletonList(extension(new FakeSecrets()).build())) + .build()); + assertTrue(profile.getPlugins().isEmpty()); + } + + @Test + public void invalidConfig() { + String[][] cases = { + {EXT + "foo = \"x\"", "unknown key \"foo\" in ext.aws_secrets_manager"}, + {EXT_TLS + "foo = \"x\"", "unknown key \"foo\" in ext.aws_secrets_manager.tls"}, + {EXT + "tls = \"x\"", "ext.aws_secrets_manager.tls must be a table, got string"}, + {EXT + "api_key_id = 1", "ext.aws_secrets_manager.api_key_id must be a string, got number"}, + { + EXT_TLS + "client_cert_id = []", + "ext.aws_secrets_manager.tls.client_cert_id must be a string, got array" + }, + { + "api_key = \"key\"\n" + EXT + "api_key_id = \"id\"", + "invalid config: API key secret ID cannot be set with an API key" + }, + { + "[profile.default.tls]\nclient_cert_data = \"x\"\nclient_key_data = \"y\"\n" + + EXT_TLS + + "client_cert_id = \"c\"\nclient_key_id = \"k\"", + "invalid config: TLS client cert secret ID cannot be set with client cert path or data; " + + "TLS client key secret ID cannot be set with client key path or data" + }, + { + "[profile.default.tls]\nserver_ca_cert_data = \"x\"\n" + + EXT_TLS + + "server_ca_cert_id = \"ca\"", + "invalid config: TLS server CA cert secret ID cannot be set with server CA cert path or" + + " data" + }, + { + "[profile.default.tls]\ndisabled = true\n" + EXT_TLS + "server_ca_cert_id = \"ca\"", + "invalid config: TLS secret IDs cannot be set when TLS is disabled" + }, + { + EXT_TLS + "client_cert_id = \"c\"", + "invalid config: TLS client cert and key secret IDs must be set together" + }, + }; + for (String[] c : cases) { + RuntimeException e = + assertThrows( + c[0], RuntimeException.class, () -> load(PROFILE + c[0] + "\n", new FakeSecrets())); + assertEquals( + c[0], "Client config extension \"aws_secrets_manager\" failed: " + c[1], e.getMessage()); + } + + // Across sources + RuntimeException e = + assertThrows( + RuntimeException.class, + () -> + load( + PROFILE + "api_key = \"key\"\n", + Collections.singletonMap("TEMPORAL_EXT_AWS_SECRETS_MANAGER_API_KEY_ID", "id"), + extension(new FakeSecrets()))); + assertTrue( + e.getMessage(), e.getMessage().endsWith("API key secret ID cannot be set with an API key")); + } + + @Test + public void apiKeySecretIsAllowedWithTlsDisabled() throws Exception { + load( + PROFILE + "[profile.default.tls]\ndisabled = true\n" + EXT + "api_key_id = \"id\"\n", + new FakeSecrets()); + } + + @Test + public void refreshIntervalAndMaxStaleness() throws Exception { + FakeSecrets secrets = new FakeSecrets(); + SecretCache cache = load(API_KEY_PROFILE, secrets).cache; + assertEquals(TimeUnit.MINUTES.toNanos(5), cache.refreshIntervalNanos); + assertEquals(TimeUnit.HOURS.toNanos(24), cache.maxStalenessNanos); + + cache = + plugin( + load( + API_KEY_PROFILE, + Collections.emptyMap(), + extension(secrets) + .setRefreshInterval(Duration.ofMinutes(1)) + .setMaxStaleness(Duration.ZERO))) + .cache; + assertEquals(TimeUnit.MINUTES.toNanos(1), cache.refreshIntervalNanos); + assertEquals(TimeUnit.HOURS.toNanos(24), cache.maxStalenessNanos); + + // Max staleness is at least the refresh interval + cache = + plugin( + load( + API_KEY_PROFILE, + Collections.emptyMap(), + extension(secrets) + .setRefreshInterval(Duration.ofHours(2)) + .setMaxStaleness(Duration.ofHours(1)))) + .cache; + assertEquals(TimeUnit.HOURS.toNanos(2), cache.maxStalenessNanos); + + RuntimeException e = + assertThrows( + RuntimeException.class, + () -> + load( + API_KEY_PROFILE, + Collections.emptyMap(), + extension(secrets).setRefreshInterval(Duration.ofSeconds(-1)))); + assertTrue(e.getMessage(), e.getMessage().endsWith("refreshInterval must not be negative")); + + // From SECRETS_MANAGER_TTL in the process env + assertEquals(0, ttlCache(secrets, "0").refreshIntervalNanos); + assertEquals(TimeUnit.SECONDS.toNanos(600), ttlCache(secrets, "600").refreshIntervalNanos); + // The option takes precedence + assertEquals( + TimeUnit.SECONDS.toNanos(1), + plugin( + load( + API_KEY_PROFILE, + Collections.emptyMap(), + extension(secrets) + .setProcessEnv(Collections.singletonMap("SECRETS_MANAGER_TTL", "600")) + .setRefreshInterval(Duration.ofSeconds(1)))) + .cache + .refreshIntervalNanos); + for (String invalid : new String[] {"-1", "1.5", "abc", "99999999999999999999"}) { + e = assertThrows(RuntimeException.class, () -> ttlCache(secrets, invalid)); + assertEquals( + "Client config extension \"aws_secrets_manager\" failed: invalid SECRETS_MANAGER_TTL \"" + + invalid + + "\", must be a non-negative whole number of seconds", + e.getMessage()); + } + } + + private static SecretCache ttlCache(FakeSecrets secrets, String ttl) throws Exception { + return plugin( + load( + API_KEY_PROFILE, + Collections.emptyMap(), + extension(secrets) + .setProcessEnv(Collections.singletonMap("SECRETS_MANAGER_TTL", ttl)))) + .cache; + } + + @Test + public void configureAppliesSecrets() throws Exception { + FakeSecrets secrets = allSecrets(); + ClientConfigProfile profile = load(ALL_PROFILE, Collections.emptyMap(), extension(secrets)); + WorkflowServiceStubsOptions options = configure(profile); + assertEquals("Bearer secret-key", authorization(options)); + assertNotNull(options.getSslContext()); + assertTrue(options.getSslContext().isClient()); + assertEquals(1, secrets.calls("api-key")); + assertEquals(1, secrets.calls("bundle")); + assertEquals(1, secrets.calls("ca")); + // An API key enables TLS by default, like an API key from envconfig + ClientConfigProfile apiKeyProfile = + load(API_KEY_PROFILE, Collections.emptyMap(), extension(new FakeSecrets().put("id", "k"))); + assertTrue(configure(apiKeyProfile).getEnableHttps()); + } + + @Test + public void configureCombinesSecretsWithProfileTls() throws Exception { + // CA from the profile, client cert and key from a secret + String toml = + PROFILE + + "[profile.default.tls]\nserver_name = \"sni\"\nserver_ca_cert_data = \"\"\"\n" + + TEST_CERT_B + + "\"\"\"\n" + + EXT_TLS + + "client_cert_id = \"bundle\"\nclient_key_id = \"bundle\"\n"; + FakeSecrets secrets = allSecrets(); + WorkflowServiceStubsOptions options = + configure(load(toml, Collections.emptyMap(), extension(secrets))); + assertNotNull(options.getSslContext()); + assertEquals(1, secrets.calls("bundle")); + + // CA from a secret, client cert and key from the profile + toml = + PROFILE + + "[profile.default.tls]\nclient_cert_data = \"\"\"\n" + + TEST_CERT_A + + "\"\"\"\nclient_key_data = \"\"\"\n" + + TEST_KEY_A + + "\"\"\"\n" + + EXT_TLS + + "server_ca_cert_id = \"ca\"\n"; + options = configure(load(toml, Collections.emptyMap(), extension(secrets))); + assertNotNull(options.getSslContext()); + assertEquals(1, secrets.calls("ca")); + } + + @Test + public void configureConflicts() throws Exception { + ClientConfigProfile profile = + load(ALL_PROFILE, Collections.emptyMap(), extension(allSecrets())); + SecretsManagerPlugin plugin = plugin(profile); + + WorkflowServiceStubsOptions.Builder withApiKey = + WorkflowServiceStubsOptions.newBuilder(profile.toWorkflowServiceStubsOptions()) + .addApiKey(() -> "key"); + IllegalStateException e = + assertThrows(IllegalStateException.class, () -> plugin.configureServiceStubs(withApiKey)); + assertEquals( + "cannot use API key from AWS Secrets Manager, API key already set", e.getMessage()); + + // An SSL context that envconfig didn't create + WorkflowServiceStubsOptions.Builder withSsl = + WorkflowServiceStubsOptions.newBuilder(profile.toWorkflowServiceStubsOptions()) + .setSslContext(GrpcSslContexts.forClient().build()); + e = assertThrows(IllegalStateException.class, () -> plugin.configureServiceStubs(withSsl)); + assertEquals( + "cannot use TLS config from AWS Secrets Manager, SSL context already set", e.getMessage()); + } + + @Test + public void configureFailsIfAFetchFails() throws Exception { + ClientConfigProfile profile = + load(API_KEY_PROFILE, Collections.emptyMap(), extension(new FakeSecrets())); + SecretCache.SecretFetchException e = + assertThrows(SecretCache.SecretFetchException.class, () -> configure(profile)); + assertEquals( + "failed fetching AWS Secrets Manager secret \"id\": no secret id", + e.getMessage().replaceAll(" \\(Service:.*", "")); + } + + @Test + public void apiKeyIsRefreshedWhenUsed() throws Exception { + FakeSecrets secrets = new FakeSecrets().put("id", "key1"); + ClientConfigProfile profile = + load( + API_KEY_PROFILE, + Collections.emptyMap(), + extension(secrets).setRefreshInterval(Duration.ZERO)); + plugin(profile).minRefreshIntervalNanos = TimeUnit.MILLISECONDS.toNanos(20); + WorkflowServiceStubsOptions options = configure(profile); + assertEquals("Bearer key1", authorization(options)); + + secrets.put("id", "key2"); + waitUntil(() -> "Bearer key2".equals(authorization(options))); + } + + @Test + public void serviceStubsRunThePlugin() throws Exception { + FakeSecrets secrets = allSecrets(); + ClientConfigProfile profile = load(ALL_PROFILE, Collections.emptyMap(), extension(secrets)); + WorkflowServiceStubs stubs = + WorkflowServiceStubs.newServiceStubs(profile.toWorkflowServiceStubsOptions()); + try { + assertEquals(1, secrets.calls("api-key")); + assertEquals(1, secrets.calls("bundle")); + assertEquals(1, secrets.calls("ca")); + } finally { + stubs.shutdownNow(); + } + } + + @Test + public void apiKeyMetricsUseFinalMetricsScope() throws Exception { + ClientConfigProfile profile = + load(API_KEY_PROFILE, Collections.emptyMap(), extension(new FakeSecrets().put("id", "k"))); + // A metrics plugin registered after this one, so its scope isn't set yet when this plugin + // configures the options + Scope scope = mock(Scope.class, RETURNS_DEEP_STUBS); + WorkflowServiceStubsPlugin metricsPlugin = + new WorkflowServiceStubsPlugin() { + @Nonnull + @Override + public String getName() { + return "test.metrics"; + } + + @Override + public void configureServiceStubs(@Nonnull WorkflowServiceStubsOptions.Builder builder) { + builder.setMetricsScope(scope); + } + + @Nonnull + @Override + public WorkflowServiceStubs connectServiceClient( + @Nonnull WorkflowServiceStubsOptions options, + @Nonnull Supplier next) { + return next.get(); + } + }; + WorkflowServiceStubs stubs = + WorkflowServiceStubs.newServiceStubs( + WorkflowServiceStubsOptions.newBuilder(profile.toWorkflowServiceStubsOptions()) + .setPlugins(plugin(profile), metricsPlugin) + .build()); + try { + verify(scope, atLeastOnce()) + .tagged( + argThat( + tags -> + Telemetry.SECRET_TYPE_API_KEY.equals(tags.get(Telemetry.LABEL_SECRET_TYPE)))); + } finally { + stubs.shutdownNow(); + } + } +} diff --git a/contrib/temporal-aws-secrets-manager/src/test/java/io/temporal/aws/secretsmanager/TestHelpers.java b/contrib/temporal-aws-secrets-manager/src/test/java/io/temporal/aws/secretsmanager/TestHelpers.java new file mode 100644 index 0000000000..ec9ecc73f8 --- /dev/null +++ b/contrib/temporal-aws-secrets-manager/src/test/java/io/temporal/aws/secretsmanager/TestHelpers.java @@ -0,0 +1,183 @@ +package io.temporal.aws.secretsmanager; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BooleanSupplier; + +final class TestHelpers { + private TestHelpers() {} + + // Self-signed EC test certs, generated just for this module's tests and deliberately expired so + // they can't be used as real credentials. They're only parsed and matched against their keys, + // never used for TLS. TEST_KEY_A_EC is TEST_KEY_A in the traditional EC format. + + static final String TEST_CERT_A = + "-----BEGIN CERTIFICATE-----\n" + + "MIIBnjCCAUWgAwIBAgIUDqZuJ4EivyOw4rVmjlePA7udGhUwCgYIKoZIzj0EAwIw\n" + + "JTEjMCEGA1UEAwwaYXdzLXNlY3JldHMtbWFuYWdlci10ZXN0LWEwHhcNMDAwMTAx\n" + + "MDAwMDAwWhcNMDAwMTAyMDAwMDAwWjAlMSMwIQYDVQQDDBphd3Mtc2VjcmV0cy1t\n" + + "YW5hZ2VyLXRlc3QtYTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABPYjw38P+CZ7\n" + + "5Gn5L4yZ7QfLq1MNax7wuqmksAu+SSwNwmjiNBM0LpvEHZGaB5uun4JVwPDX2Caw\n" + + "dOzRMMjkJfGjUzBRMB0GA1UdDgQWBBQqh6LjSKZlTFBKw8fPrKcMppVc0TAfBgNV\n" + + "HSMEGDAWgBQqh6LjSKZlTFBKw8fPrKcMppVc0TAPBgNVHRMBAf8EBTADAQH/MAoG\n" + + "CCqGSM49BAMCA0cAMEQCIBOvMndUhN/E4pi3ohOrZhZ98yUG+5wGkPirzSgiyrGA\n" + + "AiB67ITqPqGBwT1mOTKnHa0R801etROwSp8EicjU65Xi0A==\n" + + "-----END CERTIFICATE-----\n"; + + static final String TEST_KEY_A = + "-----BEGIN PRIVATE KEY-----\n" + + "MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQguBxMHVTJpKDUQdXp\n" + + "lUqPRKgtTMRVEVjYJbRybzKMYoehRANCAAT2I8N/D/gme+Rp+S+Mme0Hy6tTDWse\n" + + "8LqppLALvkksDcJo4jQTNC6bxB2Rmgebrp+CVcDw19gmsHTs0TDI5CXx\n" + + "-----END PRIVATE KEY-----\n"; + + static final String TEST_KEY_A_EC = + "-----BEGIN EC PRIVATE KEY-----\n" + + "MHcCAQEEILgcTB1UyaSg1EHV6ZVKj0SoLUzEVRFY2CW0cm8yjGKHoAoGCCqGSM49\n" + + "AwEHoUQDQgAE9iPDfw/4JnvkafkvjJntB8urUw1rHvC6qaSwC75JLA3CaOI0EzQu\n" + + "m8QdkZoHm66fglXA8NfYJrB07NEwyOQl8Q==\n" + + "-----END EC PRIVATE KEY-----\n"; + + static final String TEST_CERT_B = + "-----BEGIN CERTIFICATE-----\n" + + "MIIBoDCCAUWgAwIBAgIUJkXwBu3Peti5+WEGvCfk3YQYnFgwCgYIKoZIzj0EAwIw\n" + + "JTEjMCEGA1UEAwwaYXdzLXNlY3JldHMtbWFuYWdlci10ZXN0LWIwHhcNMDAwMTAx\n" + + "MDAwMDAwWhcNMDAwMTAyMDAwMDAwWjAlMSMwIQYDVQQDDBphd3Mtc2VjcmV0cy1t\n" + + "YW5hZ2VyLXRlc3QtYjBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABFeVEzhEhUbs\n" + + "l4sgc8vkm5Lfgemqc5h2EirK5OAna/9/Yd4feUw8JYK6VKfIeJJBVeaKjJbihaTY\n" + + "aXA9zg+PBMCjUzBRMB0GA1UdDgQWBBTBGCIL3D2zvsou4vFsR2lsmwTtdjAfBgNV\n" + + "HSMEGDAWgBTBGCIL3D2zvsou4vFsR2lsmwTtdjAPBgNVHRMBAf8EBTADAQH/MAoG\n" + + "CCqGSM49BAMCA0kAMEYCIQDnP+D+kffMeiRoerQaNplgBlaH++eFY+MPLKMLMuRc\n" + + "UAIhAKcdRK2djsmk2QP7q7yc1js9HeDONluoo2WaWLMilAyk\n" + + "-----END CERTIFICATE-----\n"; + + static final String TEST_KEY_B = + "-----BEGIN PRIVATE KEY-----\n" + + "MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgsjA2Q9nct3Mb+KOR\n" + + "akioJrOmqRw6/SzLSucTqopek9qhRANCAARXlRM4RIVG7JeLIHPL5JuS34HpqnOY\n" + + "dhIqyuTgJ2v/f2HeH3lMPCWCulSnyHiSQVXmioyW4oWk2GlwPc4PjwTA\n" + + "-----END PRIVATE KEY-----\n"; + + /** Polls until {@code cond} is true, failing if it isn't within 5 seconds. */ + static void waitUntil(BooleanSupplier cond) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (!cond.getAsBoolean()) { + if (System.nanoTime() > deadline) { + throw new AssertionError("condition not met within 5s"); + } + Thread.sleep(10); + } + } + + /** + * Returns configured values, counting fetches. A value may be an exception to throw. If {@code + * gate} is set, fetches wait for it. + */ + static final class FakeFetcher implements Fetcher { + final Map values = new ConcurrentHashMap<>(); + final Map calls = new ConcurrentHashMap<>(); + volatile CountDownLatch gate; + private final String path; + + FakeFetcher() { + this(Telemetry.FETCH_PATH_SDK); + } + + FakeFetcher(String path) { + this.path = path; + } + + void set(String secretId, String value, String versionId) { + set(secretId, value.getBytes(java.nio.charset.StandardCharsets.UTF_8), versionId); + } + + void set(String secretId, byte[] value, String versionId) { + values.put(secretId, new SecretValue(value, versionId)); + } + + void fail(String secretId, Exception err) { + values.put(secretId, err); + } + + int calls(String secretId) { + AtomicInteger n = calls.get(secretId); + return n != null ? n.get() : 0; + } + + @Override + public String path() { + return path; + } + + @Override + public SecretValue fetch(String secretId) throws Exception { + calls.computeIfAbsent(secretId, k -> new AtomicInteger()).incrementAndGet(); + CountDownLatch g = gate; + if (g != null && !g.await(5, TimeUnit.SECONDS)) { + throw new AssertionError("gate not opened"); + } + Object value = values.get(secretId); + if (value == null) { + throw new Exception("no fake value for " + secretId); + } + if (value instanceof Exception) { + throw (Exception) value; + } + return (SecretValue) value; + } + } + + /** A recorded metric update. */ + static final class MetricUpdate { + final String name; + final Map tags; + + MetricUpdate(String name, Map tags) { + this.name = name; + this.tags = tags; + } + } + + /** Telemetry that records metric updates. */ + static final class RecordingTelemetry extends Telemetry { + final List updates = new ArrayList<>(); + + RecordingTelemetry() { + super(null); + } + + @Override + synchronized void cacheLookup(Map tags) { + updates.add(new MetricUpdate(METRIC_CACHE_LOOKUP, tags)); + } + + @Override + synchronized void fetchFailed(Map tags) { + updates.add(new MetricUpdate(METRIC_FETCH_FAILED, tags)); + } + + @Override + synchronized void fetchLatency(Map tags, long nanos) { + updates.add(new MetricUpdate(METRIC_FETCH_LATENCY, tags)); + } + + /** Returns and clears the tags of each update of the given metric. */ + synchronized List> take(String name) { + List> taken = new ArrayList<>(); + updates.removeIf( + u -> { + if (u.name.equals(name)) { + taken.add(u.tags); + return true; + } + return false; + }); + return taken; + } + } +} diff --git a/contrib/temporal-payload-storage-s3driver-awssdkv2/build.gradle b/contrib/temporal-payload-storage-s3driver-awssdkv2/build.gradle index a6e57058cc..f0d5eca05e 100644 --- a/contrib/temporal-payload-storage-s3driver-awssdkv2/build.gradle +++ b/contrib/temporal-payload-storage-s3driver-awssdkv2/build.gradle @@ -1,9 +1,5 @@ description = '''Temporal Java SDK External Storage S3 Driver - AWS SDK v2 Client''' -ext { - awsSdkVersion = '2.31.0' -} - dependencies { api project(':temporal-payload-storage-s3driver') api platform("software.amazon.awssdk:bom:$awsSdkVersion") diff --git a/settings.gradle b/settings.gradle index a2ee73e91a..77c0d18a74 100644 --- a/settings.gradle +++ b/settings.gradle @@ -28,6 +28,8 @@ include 'temporal-workflowstreams' project(':temporal-workflowstreams').projectDir = file('contrib/temporal-workflowstreams') include 'temporal-aws-lambda' project(':temporal-aws-lambda').projectDir = file('contrib/temporal-aws-lambda') +include 'temporal-aws-secrets-manager' +project(':temporal-aws-secrets-manager').projectDir = file('contrib/temporal-aws-secrets-manager') include 'temporal-gcp-cloud-run-opentelemetry' project(':temporal-gcp-cloud-run-opentelemetry').projectDir = file('contrib/temporal-gcp-cloud-run-opentelemetry') include 'temporal-gcp-cloud-run-id' diff --git a/temporal-bom/build.gradle b/temporal-bom/build.gradle index 0bbae3e4bd..ef175819b8 100644 --- a/temporal-bom/build.gradle +++ b/temporal-bom/build.gradle @@ -10,6 +10,7 @@ dependencies { api project(':temporal-opentelemetry') api project(':temporal-opentracing') api project(':temporal-aws-lambda') + api project(':temporal-aws-secrets-manager') api project(':temporal-gcp-cloud-run-opentelemetry') api project(':temporal-gcp-cloud-run-id') api project(':temporal-remote-data-encoder') From fa6f2a82e62f4e5740c19a4d8309266e62d37bf0 Mon Sep 17 00:00:00 2001 From: johanngan Date: Sat, 14 Nov 2026 22:38:28 -0600 Subject: [PATCH 2/2] Support Temporal creds config via Secrets Manager in Lambda worker The Lambda worker now always loads its client config with the AWS Secrets Manager extension, so credentials can be configured as secret IDs with TEMPORAL_EXT_AWS_SECRETS_MANAGER_* environment variables or in temporal.toml. The extension only adds a plugin when secret IDs are configured. Functions without the AWS Parameters and Secrets Lambda Extension need to add software.amazon.awssdk:secretsmanager, since the Lambda Java runtimes don't include the AWS SDK. OtelLambdaWorkerConfigurationHelper now adds its OpenTelemetry plugin before existing service stubs plugins rather than after them, so its metrics scope is already set when the Secrets Manager plugin fetches TLS secrets while the stubs are configured. Before, those fetches reported to a no-op scope. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 + contrib/temporal-aws-lambda/README.md | 15 +++++ contrib/temporal-aws-lambda/build.gradle | 3 + .../aws/lambda/LambdaWorkerOptions.java | 3 + .../OtelLambdaWorkerConfigurationHelper.java | 25 ++++++--- .../aws/lambda/LambdaWorkerOptionsTest.java | 21 +++++++ ...elLambdaWorkerConfigurationHelperTest.java | 56 +++++++++++++++++++ 7 files changed, 117 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f91e4c14d..c734ba63b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,8 @@ to docs, or any other relevant information. `addEventGroups` on the command's options. - Added the experimental `temporal-aws-secrets-manager` module, a `temporal-envconfig` extension that loads the client API key and mTLS material from AWS Secrets Manager secret IDs. +- `temporal-aws-lambda` now loads Temporal client credentials from AWS Secrets Manager when secret IDs are configured + with `TEMPORAL_EXT_AWS_SECRETS_MANAGER_*` environment variables or in the TOML config. ### Changed - Updated Cloud API bindings to v0.27.0, including opt-in user group memberships, projects, diff --git a/contrib/temporal-aws-lambda/README.md b/contrib/temporal-aws-lambda/README.md index 0619796fdf..c1f0697f33 100644 --- a/contrib/temporal-aws-lambda/README.md +++ b/contrib/temporal-aws-lambda/README.md @@ -33,6 +33,21 @@ public final class Handler implements RequestHandler { Connection options are loaded with `temporal-envconfig` when the handler is constructed during Lambda cold start. The Lambda worker checks `TEMPORAL_CONFIG_FILE` first, then `$LAMBDA_TASK_ROOT/temporal.toml`, then `./temporal.toml`, then falls back to the envconfig defaults and Temporal environment variables. The `configure` callback, passed as the second parameter to `LambdaWorker.define`, also runs during handler construction, so non-invocation configuration is prepared once and reused. +### Credentials from AWS Secrets Manager + +To keep credentials out of environment variables and the Lambda package, store them in AWS Secrets Manager and configure only the secret IDs (names or ARNs). The [`temporal-aws-secrets-manager`](../temporal-aws-secrets-manager) envconfig extension is always enabled, so no code changes are needed: + +| Environment variable | TOML (under `profile..ext.aws_secrets_manager`) | +|---|---| +| `TEMPORAL_EXT_AWS_SECRETS_MANAGER_API_KEY_ID` | `api_key_id` | +| `TEMPORAL_EXT_AWS_SECRETS_MANAGER_TLS_CLIENT_CERT_ID` | `tls.client_cert_id` | +| `TEMPORAL_EXT_AWS_SECRETS_MANAGER_TLS_CLIENT_KEY_ID` | `tls.client_key_id` | +| `TEMPORAL_EXT_AWS_SECRETS_MANAGER_TLS_SERVER_CA_CERT_ID` | `tls.server_ca_cert_id` | + +Secrets are fetched through the [AWS Parameters and Secrets Lambda Extension](https://docs.aws.amazon.com/secretsmanager/latest/userguide/retrieving-secrets_lambda.html) if the function has it. Otherwise they are fetched directly with the AWS SDK for Java 2.x, an optional dependency that you add yourself: add `software.amazon.awssdk:secretsmanager` to your function's dependencies. Either way they are cached across warm invocations. The Lambda execution role needs `secretsmanager:GetSecretValue` on the configured secrets. Client keys must be PKCS#8, as for envconfig. See the [`temporal-aws-secrets-manager` README](../temporal-aws-secrets-manager) for the full set of options, validation rules, caching behavior, and observability metrics. + +When secret IDs are configured, the extension adds a plugin to the service stubs options. To add your own service stubs plugins, keep the existing ones rather than replacing them with `setPlugins`. + Use the per-invocation configuration overload when final options depend on the Lambda `Context` or on resources opened for one invocation. The cold-start `configure` callback passed as the second parameter to `LambdaWorker.define` still runs once; the invocation callback runs before Temporal service stubs, client, and worker are created: ```java diff --git a/contrib/temporal-aws-lambda/build.gradle b/contrib/temporal-aws-lambda/build.gradle index de4203109b..bd56a83380 100644 --- a/contrib/temporal-aws-lambda/build.gradle +++ b/contrib/temporal-aws-lambda/build.gradle @@ -17,6 +17,7 @@ dependencies { api project(':temporal-opentelemetry') implementation project(':temporal-envconfig') + implementation project(':temporal-aws-secrets-manager') api "io.opentelemetry:opentelemetry-api" implementation "io.opentelemetry.contrib:opentelemetry-aws-xray:$otelVersion" implementation "org.slf4j:slf4j-api:$slf4jVersion" @@ -24,6 +25,8 @@ dependencies { testImplementation project(':temporal-sdk') testImplementation "io.opentelemetry:opentelemetry-sdk" testImplementation "junit:junit:${junitVersion}" + testImplementation platform("software.amazon.awssdk:bom:$awsSdkVersion") + testImplementation "software.amazon.awssdk:secretsmanager" testRuntimeOnly group: 'ch.qos.logback', name: 'logback-classic', version: "${logbackVersion}" } diff --git a/contrib/temporal-aws-lambda/src/main/java/io/temporal/aws/lambda/LambdaWorkerOptions.java b/contrib/temporal-aws-lambda/src/main/java/io/temporal/aws/lambda/LambdaWorkerOptions.java index 7a74309368..959b6b8d88 100644 --- a/contrib/temporal-aws-lambda/src/main/java/io/temporal/aws/lambda/LambdaWorkerOptions.java +++ b/contrib/temporal-aws-lambda/src/main/java/io/temporal/aws/lambda/LambdaWorkerOptions.java @@ -1,6 +1,7 @@ package io.temporal.aws.lambda; import io.temporal.activity.DynamicActivity; +import io.temporal.aws.secretsmanager.SecretsManagerExtension; import io.temporal.client.WorkflowClientOptions; import io.temporal.common.VersioningBehavior; import io.temporal.common.WorkerDeploymentVersion; @@ -112,6 +113,8 @@ static Builder newBuilderFromEnvironment(Map env, File cwd) thro LoadClientConfigProfileOptions.newBuilder() .setConfigFilePath(resolveConfigFilePath(env, cwd)) .setEnvOverrides(env) + .setExtensions( + Collections.singletonList(SecretsManagerExtension.getDefaultInstance())) .build()); return new Builder(profile, env); } diff --git a/contrib/temporal-aws-lambda/src/main/java/io/temporal/aws/lambda/OtelLambdaWorkerConfigurationHelper.java b/contrib/temporal-aws-lambda/src/main/java/io/temporal/aws/lambda/OtelLambdaWorkerConfigurationHelper.java index d087b1db49..2aef6d34e6 100644 --- a/contrib/temporal-aws-lambda/src/main/java/io/temporal/aws/lambda/OtelLambdaWorkerConfigurationHelper.java +++ b/contrib/temporal-aws-lambda/src/main/java/io/temporal/aws/lambda/OtelLambdaWorkerConfigurationHelper.java @@ -8,7 +8,6 @@ import io.temporal.serviceclient.WorkflowServiceStubsOptions; import io.temporal.serviceclient.WorkflowServiceStubsPlugin; import java.time.Duration; -import java.util.Arrays; import java.util.Map; import java.util.Objects; import java.util.function.Consumer; @@ -53,6 +52,11 @@ public static void configure(@Nonnull LambdaWorkerOptions.Builder options) { *

By default this method creates an OpenTelemetry SDK with OTLP trace and metric exporters and * AWS X-Ray-compatible trace ID generation. If {@link Builder#setOpenTelemetry(OpenTelemetry)} is * used, the provided instance is used instead and exporters are not created. + * + *

The OpenTelemetry plugin is added before any service stubs plugins already in the options, + * so that its metrics scope is set before they run. For example, the AWS Secrets Manager + * extension's plugin fetches TLS secrets while the service stubs are configured, and reports + * those fetches to the metrics scope set at that point. */ public static void configure( @Nonnull LambdaWorkerOptions.Builder options, @Nonnull Consumer configure) { @@ -213,7 +217,7 @@ void apply(LambdaWorkerOptions.Builder options) { Objects.requireNonNull(options, "options"); applyLambdaDefaults(); OpenTelemetryPlugin plugin = delegate.build(); - appendServiceStubsPlugin(options.getWorkflowServiceStubsOptionsBuilder(), plugin); + prependServiceStubsPlugin(options.getWorkflowServiceStubsOptionsBuilder(), plugin); options.addShutdownHook(plugin.newFlushHook()); } @@ -254,15 +258,20 @@ OpenTelemetry create( IdGenerator idGenerator); } - private static void appendServiceStubsPlugin( + /** + * Adds the plugin before the existing ones. Plugins configure the service stubs in order, so its + * metrics scope is set before the others run, for example before the AWS Secrets Manager + * extension's plugin reports its TLS secret fetches. + */ + private static void prependServiceStubsPlugin( WorkflowServiceStubsOptions.Builder options, WorkflowServiceStubsPlugin plugin) { WorkflowServiceStubsPlugin[] existing = options.build().getPlugins(); int existingLength = existing == null ? 0 : existing.length; - WorkflowServiceStubsPlugin[] plugins = - existingLength == 0 - ? new WorkflowServiceStubsPlugin[1] - : Arrays.copyOf(existing, existingLength + 1); - plugins[existingLength] = Objects.requireNonNull(plugin, "plugin"); + WorkflowServiceStubsPlugin[] plugins = new WorkflowServiceStubsPlugin[existingLength + 1]; + plugins[0] = Objects.requireNonNull(plugin, "plugin"); + if (existingLength > 0) { + System.arraycopy(existing, 0, plugins, 1, existingLength); + } options.setPlugins(plugins); } } diff --git a/contrib/temporal-aws-lambda/src/test/java/io/temporal/aws/lambda/LambdaWorkerOptionsTest.java b/contrib/temporal-aws-lambda/src/test/java/io/temporal/aws/lambda/LambdaWorkerOptionsTest.java index 39c9cd6547..5c5051f2c3 100644 --- a/contrib/temporal-aws-lambda/src/test/java/io/temporal/aws/lambda/LambdaWorkerOptionsTest.java +++ b/contrib/temporal-aws-lambda/src/test/java/io/temporal/aws/lambda/LambdaWorkerOptionsTest.java @@ -7,6 +7,7 @@ import io.temporal.common.VersioningBehavior; import io.temporal.common.WorkerDeploymentVersion; import io.temporal.common.converter.EncodedValues; +import io.temporal.serviceclient.WorkflowServiceStubsPlugin; import io.temporal.worker.WorkerDeploymentOptions; import io.temporal.worker.WorkerFactoryOptions; import io.temporal.worker.WorkerOptions; @@ -310,6 +311,26 @@ public void dynamicRegistrationMethodsRejectNullInputs() throws IOException { NullPointerException.class, () -> options.registerDynamicActivityImplementation(null)); } + @Test + public void secretsManagerPluginIsAddedWhenSecretIdsAreConfigured() throws IOException { + Map env = baseEnv(); + env.put("TEMPORAL_EXT_AWS_SECRETS_MANAGER_API_KEY_ID", "temporal/api-key"); + WorkflowServiceStubsPlugin[] plugins = + LambdaWorkerOptions.newBuilderFromEnvironment(env) + .getWorkflowServiceStubsOptionsBuilder() + .build() + .getPlugins(); + assertEquals(1, plugins.length); + assertEquals("io.temporal.aws.secretsmanager", plugins[0].getName()); + + WorkflowServiceStubsPlugin[] noPlugins = + LambdaWorkerOptions.newBuilderFromEnvironment(baseEnv()) + .getWorkflowServiceStubsOptionsBuilder() + .build() + .getPlugins(); + assertTrue(noPlugins == null || noPlugins.length == 0); + } + private Map baseEnv() { Map env = new HashMap<>(); env.put(LambdaWorkerOptions.TEMPORAL_CONFIG_FILE, "/nonexistent/temporal.toml"); diff --git a/contrib/temporal-aws-lambda/src/test/java/io/temporal/aws/lambda/OtelLambdaWorkerConfigurationHelperTest.java b/contrib/temporal-aws-lambda/src/test/java/io/temporal/aws/lambda/OtelLambdaWorkerConfigurationHelperTest.java index 3adac8a55a..ec676a4c2b 100644 --- a/contrib/temporal-aws-lambda/src/test/java/io/temporal/aws/lambda/OtelLambdaWorkerConfigurationHelperTest.java +++ b/contrib/temporal-aws-lambda/src/test/java/io/temporal/aws/lambda/OtelLambdaWorkerConfigurationHelperTest.java @@ -4,6 +4,8 @@ import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.RequestHandler; +import com.uber.m3.tally.NoopScope; +import com.uber.m3.tally.Scope; import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.metrics.MeterBuilder; import io.opentelemetry.api.metrics.MeterProvider; @@ -17,6 +19,8 @@ import io.temporal.common.WorkerDeploymentVersion; import io.temporal.opentelemetry.OpenTelemetryPlugin; import io.temporal.opentelemetry.TimedShutdownHook; +import io.temporal.serviceclient.WorkflowServiceStubs; +import io.temporal.serviceclient.WorkflowServiceStubsOptions; import io.temporal.serviceclient.WorkflowServiceStubsPlugin; import java.io.Closeable; import java.time.Duration; @@ -26,6 +30,9 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; +import javax.annotation.Nonnull; import org.junit.Test; public class OtelLambdaWorkerConfigurationHelperTest { @@ -129,6 +136,55 @@ public void pluginInstallsMetricsScopeAndTracingInterceptors() throws Exception 1, options.getWorkerFactoryOptionsBuilder().build().getWorkerInterceptors().length); } + @Test + public void pluginComesBeforeExistingPluginsSoTheySeeItsMetricsScope() throws Exception { + LambdaWorkerOptions.Builder options = LambdaWorkerOptions.newBuilderFromEnvironment(baseEnv()); + // Stands in for a plugin that reports to the metrics scope while configuring the stubs, like + // the AWS Secrets Manager extension's plugin from the envconfig profile + AtomicReference seenScope = new AtomicReference<>(); + WorkflowServiceStubsPlugin existing = + new WorkflowServiceStubsPlugin() { + @Nonnull + @Override + public String getName() { + return "test.existing"; + } + + @Override + public void configureServiceStubs(@Nonnull WorkflowServiceStubsOptions.Builder builder) { + seenScope.set(builder.build().getMetricsScope()); + } + + @Nonnull + @Override + public WorkflowServiceStubs connectServiceClient( + @Nonnull WorkflowServiceStubsOptions options, + @Nonnull Supplier next) { + return next.get(); + } + }; + options.getWorkflowServiceStubsOptionsBuilder().setPlugins(existing); + + OtelLambdaWorkerConfigurationHelper.configure( + options, builder -> builder.setOpenTelemetry(OpenTelemetry.noop()).setFlushHook(() -> {})); + + WorkflowServiceStubsPlugin[] plugins = + options.getWorkflowServiceStubsOptionsBuilder().build().getPlugins(); + assertEquals(2, plugins.length); + assertTrue(plugins[0] instanceof OpenTelemetryPlugin); + assertSame(existing, plugins[1]); + + // Configure in plugin order, as WorkflowServiceStubs.newServiceStubs does + WorkflowServiceStubsOptions.Builder stubsOptions = + WorkflowServiceStubsOptions.newBuilder( + options.getWorkflowServiceStubsOptionsBuilder().build()); + for (WorkflowServiceStubsPlugin plugin : plugins) { + plugin.configureServiceStubs(stubsOptions); + } + assertNotNull(seenScope.get()); + assertFalse(seenScope.get() instanceof NoopScope); + } + @Test public void configureRegistersPluginAndPerInvocationFlushHook() throws Exception { LambdaWorkerOptions.Builder options = LambdaWorkerOptions.newBuilderFromEnvironment(baseEnv());