diff --git a/CHANGELOG.md b/CHANGELOG.md index 9153295fe1..c734ba63b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,10 @@ to docs, or any other relevant information. - Experimental Event Groups associate related workflow commands so their history events can be viewed together. Create a group with `Workflow.newEventGroup` and attach it by calling `addEventGroups` on the command's options. +- Added the experimental `temporal-aws-secrets-manager` module, a `temporal-envconfig` extension that loads the client + API key and mTLS material from AWS Secrets Manager secret IDs. +- `temporal-aws-lambda` now loads Temporal client credentials from AWS Secrets Manager when secret IDs are configured + with `TEMPORAL_EXT_AWS_SECRETS_MANAGER_*` environment variables or in the TOML config. ### Changed - Updated Cloud API bindings to v0.27.0, including opt-in user group memberships, projects, diff --git a/build.gradle b/build.gradle index feb580323b..f0b07a2d64 100644 --- a/build.gradle +++ b/build.gradle @@ -46,6 +46,9 @@ ext { jsonPathVersion = '2.9.0' // compileOnly + // api in temporal-payload-storage-s3driver-awssdkv2, compileOnly in temporal-aws-secrets-manager + awsSdkVersion = '2.31.0' + cronUtilsVersion = '9.2.1' // for test server only // This is the version of Spring Boot we use to _build the SDK_. This has to be in the diff --git a/contrib/temporal-aws-lambda/README.md b/contrib/temporal-aws-lambda/README.md index 0619796fdf..c1f0697f33 100644 --- a/contrib/temporal-aws-lambda/README.md +++ b/contrib/temporal-aws-lambda/README.md @@ -33,6 +33,21 @@ public final class Handler implements RequestHandler { Connection options are loaded with `temporal-envconfig` when the handler is constructed during Lambda cold start. The Lambda worker checks `TEMPORAL_CONFIG_FILE` first, then `$LAMBDA_TASK_ROOT/temporal.toml`, then `./temporal.toml`, then falls back to the envconfig defaults and Temporal environment variables. The `configure` callback, passed as the second parameter to `LambdaWorker.define`, also runs during handler construction, so non-invocation configuration is prepared once and reused. +### Credentials from AWS Secrets Manager + +To keep credentials out of environment variables and the Lambda package, store them in AWS Secrets Manager and configure only the secret IDs (names or ARNs). The [`temporal-aws-secrets-manager`](../temporal-aws-secrets-manager) envconfig extension is always enabled, so no code changes are needed: + +| Environment variable | TOML (under `profile..ext.aws_secrets_manager`) | +|---|---| +| `TEMPORAL_EXT_AWS_SECRETS_MANAGER_API_KEY_ID` | `api_key_id` | +| `TEMPORAL_EXT_AWS_SECRETS_MANAGER_TLS_CLIENT_CERT_ID` | `tls.client_cert_id` | +| `TEMPORAL_EXT_AWS_SECRETS_MANAGER_TLS_CLIENT_KEY_ID` | `tls.client_key_id` | +| `TEMPORAL_EXT_AWS_SECRETS_MANAGER_TLS_SERVER_CA_CERT_ID` | `tls.server_ca_cert_id` | + +Secrets are fetched through the [AWS Parameters and Secrets Lambda Extension](https://docs.aws.amazon.com/secretsmanager/latest/userguide/retrieving-secrets_lambda.html) if the function has it. Otherwise they are fetched directly with the AWS SDK for Java 2.x, an optional dependency that you add yourself: add `software.amazon.awssdk:secretsmanager` to your function's dependencies. Either way they are cached across warm invocations. The Lambda execution role needs `secretsmanager:GetSecretValue` on the configured secrets. Client keys must be PKCS#8, as for envconfig. See the [`temporal-aws-secrets-manager` README](../temporal-aws-secrets-manager) for the full set of options, validation rules, caching behavior, and observability metrics. + +When secret IDs are configured, the extension adds a plugin to the service stubs options. To add your own service stubs plugins, keep the existing ones rather than replacing them with `setPlugins`. + Use the per-invocation configuration overload when final options depend on the Lambda `Context` or on resources opened for one invocation. The cold-start `configure` callback passed as the second parameter to `LambdaWorker.define` still runs once; the invocation callback runs before Temporal service stubs, client, and worker are created: ```java diff --git a/contrib/temporal-aws-lambda/build.gradle b/contrib/temporal-aws-lambda/build.gradle index de4203109b..bd56a83380 100644 --- a/contrib/temporal-aws-lambda/build.gradle +++ b/contrib/temporal-aws-lambda/build.gradle @@ -17,6 +17,7 @@ dependencies { api project(':temporal-opentelemetry') implementation project(':temporal-envconfig') + implementation project(':temporal-aws-secrets-manager') api "io.opentelemetry:opentelemetry-api" implementation "io.opentelemetry.contrib:opentelemetry-aws-xray:$otelVersion" implementation "org.slf4j:slf4j-api:$slf4jVersion" @@ -24,6 +25,8 @@ dependencies { testImplementation project(':temporal-sdk') testImplementation "io.opentelemetry:opentelemetry-sdk" testImplementation "junit:junit:${junitVersion}" + testImplementation platform("software.amazon.awssdk:bom:$awsSdkVersion") + testImplementation "software.amazon.awssdk:secretsmanager" testRuntimeOnly group: 'ch.qos.logback', name: 'logback-classic', version: "${logbackVersion}" } diff --git a/contrib/temporal-aws-lambda/src/main/java/io/temporal/aws/lambda/LambdaWorkerOptions.java b/contrib/temporal-aws-lambda/src/main/java/io/temporal/aws/lambda/LambdaWorkerOptions.java index 7a74309368..959b6b8d88 100644 --- a/contrib/temporal-aws-lambda/src/main/java/io/temporal/aws/lambda/LambdaWorkerOptions.java +++ b/contrib/temporal-aws-lambda/src/main/java/io/temporal/aws/lambda/LambdaWorkerOptions.java @@ -1,6 +1,7 @@ package io.temporal.aws.lambda; import io.temporal.activity.DynamicActivity; +import io.temporal.aws.secretsmanager.SecretsManagerExtension; import io.temporal.client.WorkflowClientOptions; import io.temporal.common.VersioningBehavior; import io.temporal.common.WorkerDeploymentVersion; @@ -112,6 +113,8 @@ static Builder newBuilderFromEnvironment(Map env, File cwd) thro LoadClientConfigProfileOptions.newBuilder() .setConfigFilePath(resolveConfigFilePath(env, cwd)) .setEnvOverrides(env) + .setExtensions( + Collections.singletonList(SecretsManagerExtension.getDefaultInstance())) .build()); return new Builder(profile, env); } diff --git a/contrib/temporal-aws-lambda/src/main/java/io/temporal/aws/lambda/OtelLambdaWorkerConfigurationHelper.java b/contrib/temporal-aws-lambda/src/main/java/io/temporal/aws/lambda/OtelLambdaWorkerConfigurationHelper.java index d087b1db49..2aef6d34e6 100644 --- a/contrib/temporal-aws-lambda/src/main/java/io/temporal/aws/lambda/OtelLambdaWorkerConfigurationHelper.java +++ b/contrib/temporal-aws-lambda/src/main/java/io/temporal/aws/lambda/OtelLambdaWorkerConfigurationHelper.java @@ -8,7 +8,6 @@ import io.temporal.serviceclient.WorkflowServiceStubsOptions; import io.temporal.serviceclient.WorkflowServiceStubsPlugin; import java.time.Duration; -import java.util.Arrays; import java.util.Map; import java.util.Objects; import java.util.function.Consumer; @@ -53,6 +52,11 @@ public static void configure(@Nonnull LambdaWorkerOptions.Builder options) { *

By default this method creates an OpenTelemetry SDK with OTLP trace and metric exporters and * AWS X-Ray-compatible trace ID generation. If {@link Builder#setOpenTelemetry(OpenTelemetry)} is * used, the provided instance is used instead and exporters are not created. + * + *

The OpenTelemetry plugin is added before any service stubs plugins already in the options, + * so that its metrics scope is set before they run. For example, the AWS Secrets Manager + * extension's plugin fetches TLS secrets while the service stubs are configured, and reports + * those fetches to the metrics scope set at that point. */ public static void configure( @Nonnull LambdaWorkerOptions.Builder options, @Nonnull Consumer configure) { @@ -213,7 +217,7 @@ void apply(LambdaWorkerOptions.Builder options) { Objects.requireNonNull(options, "options"); applyLambdaDefaults(); OpenTelemetryPlugin plugin = delegate.build(); - appendServiceStubsPlugin(options.getWorkflowServiceStubsOptionsBuilder(), plugin); + prependServiceStubsPlugin(options.getWorkflowServiceStubsOptionsBuilder(), plugin); options.addShutdownHook(plugin.newFlushHook()); } @@ -254,15 +258,20 @@ OpenTelemetry create( IdGenerator idGenerator); } - private static void appendServiceStubsPlugin( + /** + * Adds the plugin before the existing ones. Plugins configure the service stubs in order, so its + * metrics scope is set before the others run, for example before the AWS Secrets Manager + * extension's plugin reports its TLS secret fetches. + */ + private static void prependServiceStubsPlugin( WorkflowServiceStubsOptions.Builder options, WorkflowServiceStubsPlugin plugin) { WorkflowServiceStubsPlugin[] existing = options.build().getPlugins(); int existingLength = existing == null ? 0 : existing.length; - WorkflowServiceStubsPlugin[] plugins = - existingLength == 0 - ? new WorkflowServiceStubsPlugin[1] - : Arrays.copyOf(existing, existingLength + 1); - plugins[existingLength] = Objects.requireNonNull(plugin, "plugin"); + WorkflowServiceStubsPlugin[] plugins = new WorkflowServiceStubsPlugin[existingLength + 1]; + plugins[0] = Objects.requireNonNull(plugin, "plugin"); + if (existingLength > 0) { + System.arraycopy(existing, 0, plugins, 1, existingLength); + } options.setPlugins(plugins); } } diff --git a/contrib/temporal-aws-lambda/src/test/java/io/temporal/aws/lambda/LambdaWorkerOptionsTest.java b/contrib/temporal-aws-lambda/src/test/java/io/temporal/aws/lambda/LambdaWorkerOptionsTest.java index 39c9cd6547..5c5051f2c3 100644 --- a/contrib/temporal-aws-lambda/src/test/java/io/temporal/aws/lambda/LambdaWorkerOptionsTest.java +++ b/contrib/temporal-aws-lambda/src/test/java/io/temporal/aws/lambda/LambdaWorkerOptionsTest.java @@ -7,6 +7,7 @@ import io.temporal.common.VersioningBehavior; import io.temporal.common.WorkerDeploymentVersion; import io.temporal.common.converter.EncodedValues; +import io.temporal.serviceclient.WorkflowServiceStubsPlugin; import io.temporal.worker.WorkerDeploymentOptions; import io.temporal.worker.WorkerFactoryOptions; import io.temporal.worker.WorkerOptions; @@ -310,6 +311,26 @@ public void dynamicRegistrationMethodsRejectNullInputs() throws IOException { NullPointerException.class, () -> options.registerDynamicActivityImplementation(null)); } + @Test + public void secretsManagerPluginIsAddedWhenSecretIdsAreConfigured() throws IOException { + Map env = baseEnv(); + env.put("TEMPORAL_EXT_AWS_SECRETS_MANAGER_API_KEY_ID", "temporal/api-key"); + WorkflowServiceStubsPlugin[] plugins = + LambdaWorkerOptions.newBuilderFromEnvironment(env) + .getWorkflowServiceStubsOptionsBuilder() + .build() + .getPlugins(); + assertEquals(1, plugins.length); + assertEquals("io.temporal.aws.secretsmanager", plugins[0].getName()); + + WorkflowServiceStubsPlugin[] noPlugins = + LambdaWorkerOptions.newBuilderFromEnvironment(baseEnv()) + .getWorkflowServiceStubsOptionsBuilder() + .build() + .getPlugins(); + assertTrue(noPlugins == null || noPlugins.length == 0); + } + private Map baseEnv() { Map env = new HashMap<>(); env.put(LambdaWorkerOptions.TEMPORAL_CONFIG_FILE, "/nonexistent/temporal.toml"); diff --git a/contrib/temporal-aws-lambda/src/test/java/io/temporal/aws/lambda/OtelLambdaWorkerConfigurationHelperTest.java b/contrib/temporal-aws-lambda/src/test/java/io/temporal/aws/lambda/OtelLambdaWorkerConfigurationHelperTest.java index 3adac8a55a..ec676a4c2b 100644 --- a/contrib/temporal-aws-lambda/src/test/java/io/temporal/aws/lambda/OtelLambdaWorkerConfigurationHelperTest.java +++ b/contrib/temporal-aws-lambda/src/test/java/io/temporal/aws/lambda/OtelLambdaWorkerConfigurationHelperTest.java @@ -4,6 +4,8 @@ import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.RequestHandler; +import com.uber.m3.tally.NoopScope; +import com.uber.m3.tally.Scope; import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.metrics.MeterBuilder; import io.opentelemetry.api.metrics.MeterProvider; @@ -17,6 +19,8 @@ import io.temporal.common.WorkerDeploymentVersion; import io.temporal.opentelemetry.OpenTelemetryPlugin; import io.temporal.opentelemetry.TimedShutdownHook; +import io.temporal.serviceclient.WorkflowServiceStubs; +import io.temporal.serviceclient.WorkflowServiceStubsOptions; import io.temporal.serviceclient.WorkflowServiceStubsPlugin; import java.io.Closeable; import java.time.Duration; @@ -26,6 +30,9 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; +import javax.annotation.Nonnull; import org.junit.Test; public class OtelLambdaWorkerConfigurationHelperTest { @@ -129,6 +136,55 @@ public void pluginInstallsMetricsScopeAndTracingInterceptors() throws Exception 1, options.getWorkerFactoryOptionsBuilder().build().getWorkerInterceptors().length); } + @Test + public void pluginComesBeforeExistingPluginsSoTheySeeItsMetricsScope() throws Exception { + LambdaWorkerOptions.Builder options = LambdaWorkerOptions.newBuilderFromEnvironment(baseEnv()); + // Stands in for a plugin that reports to the metrics scope while configuring the stubs, like + // the AWS Secrets Manager extension's plugin from the envconfig profile + AtomicReference seenScope = new AtomicReference<>(); + WorkflowServiceStubsPlugin existing = + new WorkflowServiceStubsPlugin() { + @Nonnull + @Override + public String getName() { + return "test.existing"; + } + + @Override + public void configureServiceStubs(@Nonnull WorkflowServiceStubsOptions.Builder builder) { + seenScope.set(builder.build().getMetricsScope()); + } + + @Nonnull + @Override + public WorkflowServiceStubs connectServiceClient( + @Nonnull WorkflowServiceStubsOptions options, + @Nonnull Supplier next) { + return next.get(); + } + }; + options.getWorkflowServiceStubsOptionsBuilder().setPlugins(existing); + + OtelLambdaWorkerConfigurationHelper.configure( + options, builder -> builder.setOpenTelemetry(OpenTelemetry.noop()).setFlushHook(() -> {})); + + WorkflowServiceStubsPlugin[] plugins = + options.getWorkflowServiceStubsOptionsBuilder().build().getPlugins(); + assertEquals(2, plugins.length); + assertTrue(plugins[0] instanceof OpenTelemetryPlugin); + assertSame(existing, plugins[1]); + + // Configure in plugin order, as WorkflowServiceStubs.newServiceStubs does + WorkflowServiceStubsOptions.Builder stubsOptions = + WorkflowServiceStubsOptions.newBuilder( + options.getWorkflowServiceStubsOptionsBuilder().build()); + for (WorkflowServiceStubsPlugin plugin : plugins) { + plugin.configureServiceStubs(stubsOptions); + } + assertNotNull(seenScope.get()); + assertFalse(seenScope.get() instanceof NoopScope); + } + @Test public void configureRegistersPluginAndPerInvocationFlushHook() throws Exception { LambdaWorkerOptions.Builder options = LambdaWorkerOptions.newBuilderFromEnvironment(baseEnv()); diff --git a/contrib/temporal-aws-secrets-manager/README.md b/contrib/temporal-aws-secrets-manager/README.md new file mode 100644 index 0000000000..1bc4744e81 --- /dev/null +++ b/contrib/temporal-aws-secrets-manager/README.md @@ -0,0 +1,149 @@ +# AWS Secrets Manager for Temporal Client Config + +> ⚠️ **This module is currently at an experimental release stage.** ⚠️ + +`temporal-aws-secrets-manager` is a [`temporal-envconfig`](../../temporal-envconfig) extension that loads Temporal +client credentials (API key and/or mTLS material) from [AWS Secrets Manager](https://aws.amazon.com/secrets-manager/). +Only secret IDs live in your config. Secret material is fetched at runtime and kept in memory. + +## Install + +Add `io.temporal:temporal-aws-secrets-manager` next to your Temporal SDK dependency. + +If the +[AWS Parameters and Secrets Lambda Extension](https://docs.aws.amazon.com/secretsmanager/latest/userguide/retrieving-secrets_lambda.html) +is available (see [Fetching and caching](#fetching-and-caching)), no other dependencies are needed. Otherwise, secrets +are fetched directly with the AWS SDK for Java 2.x, which is an optional dependency that you add yourself: + +```groovy +implementation "software.amazon.awssdk:secretsmanager:" +``` + +If it's needed and missing, loading the config fails with an error saying so. + +## Usage + +```java +ClientConfigProfile profile = + ClientConfigProfile.load( + LoadClientConfigProfileOptions.newBuilder() + .setExtensions(Collections.singletonList(SecretsManagerExtension.getDefaultInstance())) + .build()); +WorkflowServiceStubs service = + WorkflowServiceStubs.newServiceStubs(profile.toWorkflowServiceStubsOptions()); +WorkflowClient client = WorkflowClient.newInstance(service, profile.toWorkflowClientOptions()); +``` + +The extension adds a plugin to the profile's plugins, which are set on the options returned by +`toWorkflowServiceStubsOptions()`. The plugin fetches the secrets when service stubs are created. To use other service +stubs plugins, add them to these rather than replacing them. + +## Configuration + +Each credential is configured with a Secrets Manager secret ID (friendly name or full ARN), either in the TOML config +file or with environment variables. Environment variables take precedence over the file. + +| TOML (under `profile..ext.aws_secrets_manager`) | Environment variable | Replaces | +|---|---|---| +| `api_key_id` | `TEMPORAL_EXT_AWS_SECRETS_MANAGER_API_KEY_ID` | `api_key` / `TEMPORAL_API_KEY` | +| `tls.client_cert_id` | `TEMPORAL_EXT_AWS_SECRETS_MANAGER_TLS_CLIENT_CERT_ID` | `tls.client_cert_{path,data}` / `TEMPORAL_TLS_CLIENT_CERT_{PATH,DATA}` | +| `tls.client_key_id` | `TEMPORAL_EXT_AWS_SECRETS_MANAGER_TLS_CLIENT_KEY_ID` | `tls.client_key_{path,data}` / `TEMPORAL_TLS_CLIENT_KEY_{PATH,DATA}` | +| `tls.server_ca_cert_id` | `TEMPORAL_EXT_AWS_SECRETS_MANAGER_TLS_SERVER_CA_CERT_ID` | `tls.server_ca_cert_{path,data}` / `TEMPORAL_TLS_SERVER_CA_CERT_{PATH,DATA}` | + +For example: + +```toml +[profile.default] +address = "my-namespace.a1b2c.tmprl.cloud:7233" +namespace = "my-namespace.a1b2c" + +[profile.default.ext.aws_secrets_manager.tls] +client_cert_id = "temporal/client-cert-and-key" +client_key_id = "temporal/client-cert-and-key" +``` + +Rules: + +- A secret ID can't be combined with the credential it replaces, from any source. For example, `api_key` in the file + with `TEMPORAL_EXT_AWS_SECRETS_MANAGER_API_KEY_ID` in the environment is an error. +- The client cert and key IDs must be set together. They may be the same ID, in which case the secret holds both the + PEM certificate and the PEM key. This lets the pair be rotated atomically. +- TLS secret IDs can't be used when TLS is disabled. +- An API key secret enables TLS by default, as `api_key` does. +- Secret values are used as-is: the API key as a string, and TLS material as PEM. Both `SecretString` and + `SecretBinary` secrets are supported. +- As with envconfig's `tls.client_key_{path,data}`, the client key must be PKCS#8 (`-----BEGIN PRIVATE KEY-----`). + Convert other formats with `openssl pkcs8 -topk8 -nocrypt -in key.pem -out key-pkcs8.pem`. +- TLS secret IDs can't be used with an SSL context that you set on the service stubs options yourself. +- Unknown keys under `ext.aws_secrets_manager` are rejected. + +## Fetching and caching + +When `SecretsManagerExtension.Builder.setSecretsClient` isn't used, secrets are fetched through the +[AWS Parameters and Secrets Lambda Extension](https://docs.aws.amazon.com/secretsmanager/latest/userguide/retrieving-secrets_lambda.html) +if it is present. Otherwise they are fetched directly from Secrets Manager with a client built from the default AWS +config. + +The Lambda extension is detected when the config is loaded, as follows: + +- If `PARAMETERS_SECRETS_EXTENSION_HTTP_PORT` is set to a port, the extension is used on that port. +- If it is unset and the code is running in Lambda (`AWS_LAMBDA_FUNCTION_NAME` is set), the extension's default port, + 2773, is probed, and the extension is used if something is listening there. Outside Lambda, the port is not probed + and secrets are fetched directly. +- Setting `PARAMETERS_SECRETS_EXTENSION_HTTP_PORT` to an empty value explicitly disables detection so that secrets are + always fetched directly. Use this if another service in your function listens on port 2773. + +To use your own client, for example with a specific region or credentials, set it on the extension: + +```java +SecretsManagerExtension extension = + SecretsManagerExtension.newBuilder() + .setSecretsClient(SecretsManagerClient.builder().region(Region.US_WEST_2).build()) + .build(); +``` + +Secrets are first fetched when service stubs are created, and creating them fails if a fetch fails. After that: + +- Secrets are cached by the plugin that the extension adds to the loaded profile, so service stubs created with the + same options share the cache (for example, across warm Lambda invocations). +- The API key is refreshed in the background while the service stubs are in use, so a rotated API key is picked up + without creating new stubs. TLS material is fetched only when service stubs are created. +- The refresh interval defaults to the `SECRETS_MANAGER_TTL` environment variable used by the Lambda extension (whole + seconds, where 0 disables caching), or 5 minutes if that's unset. Use `setRefreshInterval` to override it. +- If a refresh fails, the last fetched value keeps being used for up to `setMaxStaleness` (default 24 hours). + +## Observability + +The extension logs with SLF4J under `io.temporal.aws.secretsmanager`, and emits metrics to the service stubs options' +metrics scope. + +TLS secrets are fetched while the service stubs are configured, since the SSL context has to be set on their options, +so their metrics go to the metrics scope already set at that point. Set the metrics scope on the service stubs options +directly, or add a plugin that sets it, like `OpenTelemetryPlugin`, before the plugins from the envconfig profile. +Otherwise, TLS fetches emit no metrics. `OtelLambdaWorkerConfigurationHelper` in `temporal-aws-lambda` already adds its +plugin first. + +Metrics are all labeled with `secret_type` (`api_key`, `tls_client_cert`, `tls_server_ca_cert`; client cert and client +key metrics are both labeled `tls_client_cert`) and `fetch_path` (`extension` or `sdk`). Currently, the following +metrics are emitted: + +| Metric | Type | Extra labels | Description | +|---|---|---|---| +| `temporal_aws_secrets_manager_fetch_failed` | Counter | `failure_reason`, `fallback` (`stale`, `none`) | Failed fetches, and whether the previous value was used instead | +| `temporal_aws_secrets_manager_fetch_latency` | Timer | `outcome` (`success`, `failure`) | Latency of each fetch | +| `temporal_aws_secrets_manager_cache_lookup` | Counter | `result` (`hit`, `miss`, `miss_with_fallback`) | Lookups that returned a value | + +## Required IAM permissions + +The AWS credentials in use (for example, the Lambda execution role) need `secretsmanager:GetSecretValue` on each +configured secret: + +```json +{ + "Effect": "Allow", + "Action": "secretsmanager:GetSecretValue", + "Resource": "arn:aws:secretsmanager:us-west-2:123456789012:secret:temporal/*" +} +``` + +If a secret is encrypted with a customer-managed KMS key, `kms:Decrypt` on that key is also required. diff --git a/contrib/temporal-aws-secrets-manager/build.gradle b/contrib/temporal-aws-secrets-manager/build.gradle new file mode 100644 index 0000000000..4e88bb67f7 --- /dev/null +++ b/contrib/temporal-aws-secrets-manager/build.gradle @@ -0,0 +1,23 @@ +description = '''Temporal Java SDK AWS Secrets Manager Client Config Extension''' + +dependencies { + // This module shouldn't carry temporal-sdk with it, especially for situations when users may + // be using a shaded artifact. + compileOnly project(':temporal-sdk') + + api project(':temporal-envconfig') + implementation "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}" + implementation "org.slf4j:slf4j-api:$slf4jVersion" + + // Optional: only needed to fetch secrets directly from Secrets Manager, when the AWS + // Parameters and Secrets Lambda Extension isn't available. Users add it themselves. + compileOnly platform("software.amazon.awssdk:bom:$awsSdkVersion") + compileOnly "software.amazon.awssdk:secretsmanager" + + testImplementation project(':temporal-sdk') + testImplementation platform("software.amazon.awssdk:bom:$awsSdkVersion") + testImplementation "software.amazon.awssdk:secretsmanager" + testImplementation "junit:junit:${junitVersion}" + testImplementation "org.mockito:mockito-core:${mockitoVersion}" + testRuntimeOnly group: 'ch.qos.logback', name: 'logback-classic', version: "${logbackVersion}" +} diff --git a/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/FetchErrors.java b/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/FetchErrors.java new file mode 100644 index 0000000000..317c821654 --- /dev/null +++ b/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/FetchErrors.java @@ -0,0 +1,47 @@ +package io.temporal.aws.secretsmanager; + +/** Errors from fetching secrets, classified by {@link Telemetry#classifyFailure(Throwable)}. */ +final class FetchErrors { + private FetchErrors() {} + + /** A response that was received but has no usable secret value. */ + static final class InvalidResponseException extends Exception { + InvalidResponseException(String message) { + super(message); + } + + InvalidResponseException(String message, Throwable cause) { + super(message, cause); + } + } + + /** A non-200 response from the Lambda extension. */ + static final class LambdaExtensionStatusException extends Exception { + final int status; + + LambdaExtensionStatusException(int status, String body) { + super("AWS Parameters and Secrets Lambda Extension returned status " + status + ": " + body); + this.status = status; + } + } + + /** A fetch that took longer than the fetch timeout. */ + static final class FetchTimeoutException extends Exception { + FetchTimeoutException(String message, Throwable cause) { + super(message, cause); + } + } + + /** + * An error response from Secrets Manager, with its AWS error code. Wraps the AWS SDK's exception + * so that it can be classified without loading AWS SDK classes. + */ + static final class AwsErrorException extends Exception { + final String errorCode; + + AwsErrorException(String errorCode, Throwable cause) { + super(cause.getMessage(), cause); + this.errorCode = errorCode; + } + } +} diff --git a/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/Fetcher.java b/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/Fetcher.java new file mode 100644 index 0000000000..5889a826f4 --- /dev/null +++ b/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/Fetcher.java @@ -0,0 +1,116 @@ +package io.temporal.aws.secretsmanager; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.nio.charset.StandardCharsets; +import java.util.Map; + +/** Fetches secret values by ID. Must be safe to call from multiple threads. */ +interface Fetcher { + String ENV_LAMBDA_EXTENSION_PORT = "PARAMETERS_SECRETS_EXTENSION_HTTP_PORT"; + String ENV_LAMBDA_FUNCTION_NAME = "AWS_LAMBDA_FUNCTION_NAME"; + String LAMBDA_EXTENSION_HOST = "localhost"; + int DEFAULT_LAMBDA_EXTENSION_PORT = 2773; + + /** Bounds each request made by the fetchers this package creates. */ + long FETCH_TIMEOUT_MILLIS = 10_000; + + /** The fetch_path metric label for this fetcher. */ + String path(); + + /** Fetches the current value of the given secret. */ + SecretValue fetch(String secretId) throws Exception; + + /** A fetched secret value. */ + final class SecretValue { + final byte[] value; + final String versionId; + + SecretValue(byte[] value, String versionId) { + this.value = value; + this.versionId = versionId; + } + + /** A secret value from a GetSecretValue response's fields, preferring SecretString. */ + static SecretValue fromResponse(String secretString, byte[] secretBinary, String versionId) + throws FetchErrors.InvalidResponseException { + byte[] value; + if (secretString != null) { + value = secretString.getBytes(StandardCharsets.UTF_8); + } else if (secretBinary != null && secretBinary.length > 0) { + value = secretBinary; + } else { + throw new FetchErrors.InvalidResponseException("secret has no value"); + } + return new SecretValue(value, versionId != null ? versionId : ""); + } + } + + /** + * Returns a fetcher for the Lambda extension if it is detected, and otherwise an SDK fetcher with + * a client created from the default AWS config. + * + *

The extension doesn't set PARAMETERS_SECRETS_EXTENSION_HTTP_PORT in the function's + * environment unless the user configured it, so if it is unset and running in Lambda, the + * extension's default port is probed. Setting it to empty disables detection, for when another + * service uses the default port. + */ + static Fetcher newDefaultFetcher(Map processEnv) { + String port = processEnv.get(ENV_LAMBDA_EXTENSION_PORT); + if (port == null) { + // Only try to auto-detect the Lambda extension if we're in a Lambda function + String functionName = processEnv.get(ENV_LAMBDA_FUNCTION_NAME); + if (functionName == null + || functionName.isEmpty() + || !probe(LAMBDA_EXTENSION_HOST, DEFAULT_LAMBDA_EXTENSION_PORT)) { + return newDefaultSdkFetcher(); + } + port = String.valueOf(DEFAULT_LAMBDA_EXTENSION_PORT); + } + if (port.isEmpty()) { + return newDefaultSdkFetcher(); + } + return new LambdaExtensionFetcher(port, processEnv); + } + + /** Whether something accepts connections on the given port. */ + static boolean probe(String host, int port) { + // Probing a local port is either accepted or refused almost immediately, so the timeout only + // bounds pathological cases + try (Socket socket = new Socket()) { + socket.connect(new InetSocketAddress(host, port), 100); + return true; + } catch (IOException e) { + return false; + } + } + + /** + * Whether the AWS SDK Secrets Manager client can be loaded. Checked without loading {@link + * SdkFetcher}, which references AWS SDK classes. + */ + static boolean isAwsSdkAvailable() { + try { + Class.forName( + "software.amazon.awssdk.services.secretsmanager.SecretsManagerClient", + false, + Fetcher.class.getClassLoader()); + return true; + } catch (ClassNotFoundException | LinkageError e) { + return false; + } + } + + static Fetcher newDefaultSdkFetcher() { + if (!isAwsSdkAvailable()) { + throw new IllegalStateException( + "fetching secrets directly from AWS Secrets Manager requires the AWS SDK, since the AWS " + + "Parameters and Secrets Lambda Extension was not detected (see " + + ENV_LAMBDA_EXTENSION_PORT + + "). Add a dependency on software.amazon.awssdk:secretsmanager, or add the extension " + + "to your Lambda function"); + } + return SdkFetcher.create(null); + } +} diff --git a/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/LambdaExtensionFetcher.java b/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/LambdaExtensionFetcher.java new file mode 100644 index 0000000000..6385274cd0 --- /dev/null +++ b/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/LambdaExtensionFetcher.java @@ -0,0 +1,129 @@ +package io.temporal.aws.secretsmanager; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.net.HttpURLConnection; +import java.net.Proxy; +import java.net.SocketTimeoutException; +import java.net.URL; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.Map; + +/** + * Fetches secrets through the AWS Parameters and Secrets Lambda Extension. See + * https://docs.aws.amazon.com/secretsmanager/latest/userguide/retrieving-secrets_lambda.html. + */ +final class LambdaExtensionFetcher implements Fetcher { + private static final String ENV_AWS_SESSION_TOKEN = "AWS_SESSION_TOKEN"; + private static final String TOKEN_HEADER = "X-Aws-Parameters-Secrets-Token"; + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private final String endpoint; + private final Map processEnv; + + LambdaExtensionFetcher(String port, Map processEnv) { + this.endpoint = + "http://" + LAMBDA_EXTENSION_HOST + ":" + port + "/secretsmanager/get?secretId="; + this.processEnv = processEnv; + } + + @Override + public String path() { + return Telemetry.FETCH_PATH_EXTENSION; + } + + @Override + public SecretValue fetch(String secretId) throws Exception { + URL url = new URL(endpoint + urlEncode(secretId)); + // No proxy, so that proxy settings never apply to requests to the local extension + HttpURLConnection conn = (HttpURLConnection) url.openConnection(Proxy.NO_PROXY); + conn.setConnectTimeout((int) FETCH_TIMEOUT_MILLIS); + conn.setReadTimeout((int) FETCH_TIMEOUT_MILLIS); + String token = processEnv.get(ENV_AWS_SESSION_TOKEN); + conn.setRequestProperty(TOKEN_HEADER, token != null ? token : ""); + int status; + String body; + try { + status = conn.getResponseCode(); + body = readAll(status >= 400 ? conn.getErrorStream() : conn.getInputStream()); + } catch (SocketTimeoutException e) { + throw new FetchErrors.FetchTimeoutException( + "AWS Secrets Manager secret fetch timed out after " + FETCH_TIMEOUT_MILLIS + "ms", e); + } + if (status != 200) { + throw new FetchErrors.LambdaExtensionStatusException(status, body); + } + return parseResponse(body); + } + + static SecretValue parseResponse(String body) throws FetchErrors.InvalidResponseException { + JsonNode out; + try { + // Same shape as the GetSecretValue response, with SecretBinary base64 + out = MAPPER.readTree(body); + } catch (IOException e) { + throw invalid(e.getMessage(), e); + } + if (out == null || !out.isObject()) { + throw invalid("not an object", null); + } + String secretString = textField(out, "SecretString"); + String secretBinary = textField(out, "SecretBinary"); + String versionId = textField(out, "VersionId"); + byte[] binary = null; + if (secretBinary != null && !secretBinary.isEmpty()) { + try { + binary = Base64.getDecoder().decode(secretBinary); + } catch (IllegalArgumentException e) { + throw invalid("invalid SecretBinary: " + e.getMessage(), e); + } + } + return SecretValue.fromResponse(secretString, binary, versionId); + } + + private static String textField(JsonNode out, String name) + throws FetchErrors.InvalidResponseException { + JsonNode field = out.get(name); + if (field == null || field.isNull()) { + return null; + } + if (!field.isTextual()) { + throw invalid(name + " must be a string", null); + } + return field.asText(); + } + + private static FetchErrors.InvalidResponseException invalid(String message, Throwable cause) { + return new FetchErrors.InvalidResponseException( + "failed decoding AWS Parameters and Secrets Lambda Extension response: " + message, cause); + } + + private static String urlEncode(String s) { + try { + return URLEncoder.encode(s, "UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new IllegalStateException(e); + } + } + + private static String readAll(InputStream in) throws IOException { + if (in == null) { + return ""; + } + try (InputStream stream = in) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buf = new byte[8192]; + int n; + while ((n = stream.read(buf)) != -1) { + out.write(buf, 0, n); + } + return new String(out.toByteArray(), StandardCharsets.UTF_8); + } + } +} diff --git a/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/SdkFetcher.java b/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/SdkFetcher.java new file mode 100644 index 0000000000..dbb8c7a867 --- /dev/null +++ b/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/SdkFetcher.java @@ -0,0 +1,76 @@ +package io.temporal.aws.secretsmanager; + +import java.time.Duration; +import javax.annotation.Nullable; +import software.amazon.awssdk.awscore.AwsRequestOverrideConfiguration; +import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.core.SdkBytes; +import software.amazon.awssdk.core.exception.ApiCallAttemptTimeoutException; +import software.amazon.awssdk.core.exception.ApiCallTimeoutException; +import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient; +import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueRequest; +import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueResponse; + +/** + * Fetches secrets directly from Secrets Manager with the AWS SDK, which is an optional dependency. + * This is the only class that references AWS SDK classes, so it must only be loaded when the AWS + * SDK is available. + */ +final class SdkFetcher implements Fetcher { + private static final AwsRequestOverrideConfiguration TIMEOUT = + AwsRequestOverrideConfiguration.builder() + .apiCallTimeout(Duration.ofMillis(FETCH_TIMEOUT_MILLIS)) + .build(); + + private final Object lock = new Object(); + private SecretsManagerClient client; + + private SdkFetcher(@Nullable SecretsManagerClient client) { + this.client = client; + } + + /** + * Creates a fetcher with the given client, which must be a {@code SecretsManagerClient}, or with + * a client created from the default AWS config on first use if null. + */ + static SdkFetcher create(@Nullable Object client) { + return new SdkFetcher((SecretsManagerClient) client); + } + + @Override + public String path() { + return Telemetry.FETCH_PATH_SDK; + } + + @Override + public SecretValue fetch(String secretId) throws Exception { + GetSecretValueResponse resp; + try { + resp = + getClient() + .getSecretValue( + GetSecretValueRequest.builder() + .secretId(secretId) + .overrideConfiguration(TIMEOUT) + .build()); + } catch (ApiCallTimeoutException | ApiCallAttemptTimeoutException e) { + throw new FetchErrors.FetchTimeoutException( + "AWS Secrets Manager secret fetch timed out after " + FETCH_TIMEOUT_MILLIS + "ms", e); + } catch (AwsServiceException e) { + throw new FetchErrors.AwsErrorException( + e.awsErrorDetails() != null ? e.awsErrorDetails().errorCode() : null, e); + } + SdkBytes binary = resp.secretBinary(); + return SecretValue.fromResponse( + resp.secretString(), binary != null ? binary.asByteArray() : null, resp.versionId()); + } + + private SecretsManagerClient getClient() { + synchronized (lock) { + if (client == null) { + client = SecretsManagerClient.create(); + } + return client; + } + } +} diff --git a/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/SecretCache.java b/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/SecretCache.java new file mode 100644 index 0000000000..7c78a84650 --- /dev/null +++ b/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/SecretCache.java @@ -0,0 +1,646 @@ +package io.temporal.aws.secretsmanager; + +import java.io.ByteArrayInputStream; +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.security.GeneralSecurityException; +import java.security.KeyFactory; +import java.security.PrivateKey; +import java.security.Signature; +import java.security.cert.CertificateFactory; +import java.security.cert.X509Certificate; +import java.security.spec.PKCS8EncodedKeySpec; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.LongSupplier; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import javax.annotation.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * TTL cache of the configured credentials that falls back to the last fetched value, up to the max + * staleness old, when a refresh fails. + * + *

Fetches run on a shared pool of daemon threads, so that they aren't affected by callers timing + * out or being interrupted, and don't keep the JVM running. + */ +final class SecretCache { + private static final Logger log = LoggerFactory.getLogger(SecretCache.class); + + /** + * After a failed refresh falls back to a stale value, wait this long before trying again so that + * an outage doesn't add a fetch attempt to every credential lookup. + */ + static final long FAILED_REFRESH_RETRY_INTERVAL_NANOS = TimeUnit.SECONDS.toNanos(30); + + private static final Pattern CERT_PEM_PATTERN = + Pattern.compile( + "-----BEGIN CERTIFICATE-----\r?\n.*?-----END CERTIFICATE-----", Pattern.DOTALL); + private static final Pattern KEY_PEM_PATTERN = + Pattern.compile( + "-----BEGIN ((?:RSA |EC )?PRIVATE KEY)-----\r?\n(.*?)-----END \\1-----", Pattern.DOTALL); + // Key algorithms that PKCS#8 keys are tried as, matching Netty's key loading used by envconfig + private static final List KEY_ALGORITHMS = Arrays.asList("RSA", "DSA", "EC"); + + private static final AtomicInteger FETCH_THREAD_COUNT = new AtomicInteger(); + private static final ExecutorService FETCH_EXECUTOR = + Executors.newCachedThreadPool( + r -> { + Thread t = + new Thread( + r, + "temporal-aws-secrets-manager-fetch-" + FETCH_THREAD_COUNT.incrementAndGet()); + t.setDaemon(true); + return t; + }); + + /** Configured secret IDs. Empty strings are unset. */ + static final class SecretIds { + final String apiKey; + final String clientCert; + final String clientKey; + final String serverCaCert; + + SecretIds(String apiKey, String clientCert, String clientKey, String serverCaCert) { + this.apiKey = apiKey; + this.clientCert = clientCert; + this.clientKey = clientKey; + this.serverCaCert = serverCaCert; + } + + boolean hasTls() { + return !clientCert.isEmpty() || !clientKey.isEmpty() || !serverCaCert.isEmpty(); + } + + boolean isEmpty() { + return apiKey.isEmpty() && !hasTls(); + } + } + + /** A client cert chain and its private key. */ + static final class ClientCertPair { + final X509Certificate[] chain; + final PrivateKey key; + + ClientCertPair(X509Certificate[] chain, PrivateKey key) { + this.chain = chain; + this.key = key; + } + } + + /** A failed fetch of one or more secrets, with its failure_reason label. */ + static final class SecretFetchException extends RuntimeException { + final String reason; + + SecretFetchException(String message, String reason, @Nullable Throwable cause) { + super(message, cause); + this.reason = reason; + } + } + + /** Thrown when waiting for a fetch takes longer than the fetch timeout. */ + static final class FetchWaitTimeoutException extends RuntimeException { + FetchWaitTimeoutException(String message) { + super(message); + } + } + + @FunctionalInterface + interface Parser { + /** Parses newly fetched secrets, in refs order, into the cached value. */ + T parse(List secrets) throws FetchErrors.InvalidResponseException; + } + + static final class SecretRef { + final String id; + final String secretType; + + SecretRef(String id, String secretType) { + this.id = id; + this.secretType = secretType; + } + } + + /** + * One or more secrets that are fetched and replaced together, cached as a single parsed value. + */ + static final class CacheEntry { + /** Distinct secret IDs. refs[0]'s secret type is used as the secret_type metric label. */ + final List refs; + + final Parser parser; + + // All fields below are guarded by this + + /** The fetch in progress, or null if none. */ + CompletableFuture inFlight; + + /** In refs order, null until the first successful fetch. */ + List versionIds; + + T value; + long fetchedAt; + + /** Last failed fetch since the last successful fetch, or null if none. */ + Long failedAt; + + CacheEntry(List refs, Parser parser) { + this.refs = refs; + this.parser = parser; + } + } + + final Fetcher fetcher; + final long refreshIntervalNanos; + final long maxStalenessNanos; + + /** How long callers wait for a fetch before falling back or failing. */ + volatile long fetchTimeoutNanos = TimeUnit.MILLISECONDS.toNanos(Fetcher.FETCH_TIMEOUT_MILLIS); + + /** Monotonic nanoseconds, overridable in tests. */ + volatile LongSupplier now = System::nanoTime; + + /** Runs fetches, overridable in tests. */ + volatile Executor executor = FETCH_EXECUTOR; + + @Nullable final CacheEntry apiKey; + + /** Client cert and key, fetched and replaced together to prevent tearing on rotations. */ + @Nullable final CacheEntry clientCert; + + @Nullable final CacheEntry serverCaCert; + + SecretCache(Fetcher fetcher, SecretIds ids, long refreshIntervalNanos, long maxStalenessNanos) { + this.fetcher = fetcher; + this.refreshIntervalNanos = refreshIntervalNanos; + this.maxStalenessNanos = maxStalenessNanos; + this.apiKey = + ids.apiKey.isEmpty() + ? null + : new CacheEntry<>( + Collections.singletonList(new SecretRef(ids.apiKey, Telemetry.SECRET_TYPE_API_KEY)), + SecretCache::parseApiKey); + this.clientCert = + ids.clientCert.isEmpty() ? null : newClientCertEntry(ids.clientCert, ids.clientKey); + this.serverCaCert = ids.serverCaCert.isEmpty() ? null : newServerCaCertEntry(ids.serverCaCert); + } + + /** + * Gets an entry's value, fetching it if needed. Returned values are shared across callers and + * must not be mutated. + * + * @throws SecretFetchException if the fetch failed and there's no usable previous value + * @throws FetchWaitTimeoutException if the fetch timed out and there's no usable previous value + */ + T get(CacheEntry entry, Telemetry tel) { + long start; + boolean canUseStale; + T cached; + CompletableFuture fetch = null; + boolean startFetch = false; + String result = null; + synchronized (entry) { + start = now.getAsLong(); + boolean hasValue = entry.versionIds != null; + canUseStale = hasValue && start - entry.fetchedAt < maxStalenessNanos; + cached = entry.value; + if ((hasValue && start - entry.fetchedAt < refreshIntervalNanos) + // Another caller is refreshing, so the current value is still the latest known one + || (canUseStale && entry.inFlight != null)) { + result = "hit"; + } else if (canUseStale + && entry.failedAt != null + && start - entry.failedAt < FAILED_REFRESH_RETRY_INTERVAL_NANOS) { + result = "miss_with_fallback"; + } else { + // Wait for a fetch, starting one if none is in progress + fetch = entry.inFlight; + if (fetch == null) { + fetch = new CompletableFuture<>(); + entry.inFlight = fetch; + startFetch = true; + } + } + } + if (result != null) { + recordLookup(entry, tel, result); + return cached; + } + if (startFetch) { + startFetch(entry, fetch, start, canUseStale, tel); + } + + T value; + try { + value = fetch.get(fetchTimeoutNanos, TimeUnit.NANOSECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + if (canUseStale) { + recordLookup(entry, tel, "miss_with_fallback"); + return cached; + } + throw new FetchWaitTimeoutException( + "interrupted while waiting for AWS Secrets Manager secret fetch"); + } catch (TimeoutException e) { + if (canUseStale) { + recordLookup(entry, tel, "miss_with_fallback"); + return cached; + } + throw new FetchWaitTimeoutException( + "timed out after " + + TimeUnit.NANOSECONDS.toMillis(fetchTimeoutNanos) + + "ms waiting for AWS Secrets Manager secret fetch"); + } catch (ExecutionException e) { + if (canUseStale) { + recordLookup(entry, tel, "miss_with_fallback"); + return cached; + } + Throwable cause = e.getCause(); + if (cause instanceof SecretFetchException) { + throw (SecretFetchException) cause; + } + throw new SecretFetchException(String.valueOf(cause), "other", cause); + } + recordLookup(entry, tel, "miss"); + return value; + } + + /** + * Returns an entry's current value without waiting, and starts refreshing it in the background if + * it's due. The entry must already have a value. + * + *

The value is refreshed at most every {@code minRefreshIntervalNanos}, and is kept if + * refreshing it fails, even past the max staleness. + */ + T getNow(CacheEntry entry, long minRefreshIntervalNanos, Telemetry tel) { + long start; + boolean canUseStale; + T value; + CompletableFuture fetch = null; + synchronized (entry) { + start = now.getAsLong(); + value = entry.value; + canUseStale = entry.versionIds != null && start - entry.fetchedAt < maxStalenessNanos; + if (entry.inFlight == null + && start - entry.fetchedAt >= Math.max(refreshIntervalNanos, minRefreshIntervalNanos) + && (entry.failedAt == null + || start - entry.failedAt >= FAILED_REFRESH_RETRY_INTERVAL_NANOS)) { + fetch = new CompletableFuture<>(); + entry.inFlight = fetch; + } + } + if (fetch != null) { + startFetch(entry, fetch, start, canUseStale, tel); + } + return value; + } + + private void startFetch( + CacheEntry entry, + CompletableFuture fetch, + long start, + boolean canUseStale, + Telemetry tel) { + try { + executor.execute(() -> refresh(entry, fetch, start, canUseStale, tel)); + } catch (RejectedExecutionException e) { + // Fail this fetch, including for callers already waiting on it, so that the next caller + // starts a new one + synchronized (entry) { + entry.inFlight = null; + } + fetch.completeExceptionally( + new SecretFetchException("failed starting AWS Secrets Manager secret fetch", "other", e)); + } + } + + /** Fetches an entry's secrets and records the outcome in the cache. */ + private void refresh( + CacheEntry entry, + CompletableFuture fetch, + long start, + boolean canUseStale, + Telemetry tel) { + T value = null; + List versionIds = null; + SecretFetchException failure = null; + try { + List secrets = fetchAll(entry); + value = parse(entry, secrets); + versionIds = new ArrayList<>(secrets.size()); + for (Fetcher.SecretValue secret : secrets) { + versionIds.add(secret.versionId); + } + } catch (SecretFetchException e) { + failure = e; + } catch (RuntimeException e) { + // Shouldn't happen, but don't leave the fetch pending + failure = new SecretFetchException(e.toString(), "other", e); + } + long latencyNanos = now.getAsLong() - start; + + List prevVersionIds; + long prevFetchedAt; + boolean prevFailed; + synchronized (entry) { + prevVersionIds = entry.versionIds; + prevFetchedAt = entry.fetchedAt; + prevFailed = entry.failedAt != null; + if (failure != null) { + entry.failedAt = start; + } else { + entry.value = value; + entry.versionIds = versionIds; + entry.fetchedAt = start; + entry.failedAt = null; + } + entry.inFlight = null; + } + + // Record telemetry outside the lock, and complete the fetch last so that the fetch's metrics + // and logs are recorded by the time waiters return + try { + if (failure != null) { + recordFailure(entry, failure, latencyNanos, start - prevFetchedAt, canUseStale, tel); + } else { + recordSuccess(entry, versionIds, prevVersionIds, prevFailed, latencyNanos, tel); + } + } catch (RuntimeException e) { + log.warn("Failed recording AWS Secrets Manager telemetry", e); + } finally { + if (failure != null) { + fetch.completeExceptionally(failure); + } else { + fetch.complete(value); + } + } + } + + private List fetchAll(CacheEntry entry) { + List secrets = new ArrayList<>(entry.refs.size()); + for (SecretRef ref : entry.refs) { + try { + secrets.add(fetcher.fetch(ref.id)); + } catch (Exception e) { + throw new SecretFetchException( + "failed fetching AWS Secrets Manager secret \"" + + ref.id + + "\": " + + (e.getMessage() != null ? e.getMessage() : e.toString()), + Telemetry.classifyFailure(e), + e); + } + } + return secrets; + } + + private static T parse(CacheEntry entry, List secrets) { + try { + return entry.parser.parse(secrets); + } catch (FetchErrors.InvalidResponseException e) { + throw new SecretFetchException(e.getMessage(), "invalid_response", e); + } + } + + private Map baseTags(CacheEntry entry) { + Map tags = new HashMap<>(); + tags.put(Telemetry.LABEL_SECRET_TYPE, entry.refs.get(0).secretType); + tags.put(Telemetry.LABEL_FETCH_PATH, fetcher.path()); + return tags; + } + + private void recordLookup(CacheEntry entry, Telemetry tel, String result) { + Map tags = baseTags(entry); + tags.put(Telemetry.LABEL_RESULT, result); + tel.cacheLookup(tags); + } + + private void recordFailure( + CacheEntry entry, + SecretFetchException err, + long latencyNanos, + long ageNanos, + boolean canUseStale, + Telemetry tel) { + Map latencyTags = baseTags(entry); + latencyTags.put(Telemetry.LABEL_OUTCOME, "failure"); + tel.fetchLatency(latencyTags, latencyNanos); + Map failedTags = baseTags(entry); + failedTags.put(Telemetry.LABEL_FAILURE_REASON, err.reason); + failedTags.put(Telemetry.LABEL_FALLBACK, canUseStale ? "stale" : "none"); + tel.fetchFailed(failedTags); + if (canUseStale) { + for (SecretRef ref : entry.refs) { + log.warn( + "Failed refreshing AWS Secrets Manager secret, using previously fetched value " + + "(secret_id={}, secret_type={}, error={}, age={}, max_staleness={})", + ref.id, + ref.secretType, + err.getMessage(), + Duration.ofNanos(ageNanos), + Duration.ofNanos(maxStalenessNanos)); + } + } + } + + private void recordSuccess( + CacheEntry entry, + List versionIds, + @Nullable List prevVersionIds, + boolean prevFailed, + long latencyNanos, + Telemetry tel) { + Map tags = baseTags(entry); + tags.put(Telemetry.LABEL_OUTCOME, "success"); + tel.fetchLatency(tags, latencyNanos); + for (int i = 0; i < entry.refs.size(); i++) { + SecretRef ref = entry.refs.get(i); + if (prevVersionIds == null) { + log.info( + "Fetched AWS Secrets Manager secret " + + "(secret_id={}, secret_type={}, fetch_path={}, version_id={})", + ref.id, + ref.secretType, + fetcher.path(), + versionIds.get(i)); + } else if (prevFailed || !versionIds.get(i).equals(prevVersionIds.get(i))) { + log.info( + "Fetched updated AWS Secrets Manager secret " + + "(secret_id={}, secret_type={}, previous_version_id={}, version_id={})", + ref.id, + ref.secretType, + prevVersionIds.get(i), + versionIds.get(i)); + } + } + } + + private static String parseApiKey(List secrets) + throws FetchErrors.InvalidResponseException { + try { + return StandardCharsets.UTF_8 + .newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(secrets.get(0).value)) + .toString(); + } catch (CharacterCodingException e) { + throw new FetchErrors.InvalidResponseException("API key secret is not valid UTF-8"); + } + } + + /** + * Entry for the client cert and key, which may be the same secret if bundled. Each secret's + * relevant PEM blocks are extracted, so a bundled secret can be used as both. Keys must be + * PKCS#8, like keys in envconfig. A newly fetched pair that doesn't match (e.g. fetched partway + * through rotating separate cert and key secrets) is treated as a failed fetch, so the previous + * pair keeps being used. + */ + private static CacheEntry newClientCertEntry(String certId, String keyId) { + List refs = new ArrayList<>(); + refs.add(new SecretRef(certId, Telemetry.SECRET_TYPE_TLS_CLIENT_CERT)); + if (!keyId.equals(certId)) { + refs.add(new SecretRef(keyId, Telemetry.SECRET_TYPE_TLS_CLIENT_KEY)); + } + return new CacheEntry<>( + refs, + secrets -> { + X509Certificate[] chain = parseCerts(secrets.get(0).value, "TLS client cert", certId); + if (chain.length == 0) { + throw new FetchErrors.InvalidResponseException( + "no PEM certificate found in TLS client cert secret \"" + certId + "\""); + } + PrivateKey key = parseKey(secrets.get(secrets.size() - 1).value, keyId); + if (!keyMatchesCert(key, chain[0])) { + throw new FetchErrors.InvalidResponseException( + "TLS client cert and key from AWS Secrets Manager secrets \"" + + certId + + "\" and \"" + + keyId + + "\" don't match"); + } + return new ClientCertPair(chain, key); + }); + } + + private static CacheEntry newServerCaCertEntry(String secretId) { + return new CacheEntry<>( + Collections.singletonList( + new SecretRef(secretId, Telemetry.SECRET_TYPE_TLS_SERVER_CA_CERT)), + secrets -> { + X509Certificate[] certs = + parseCerts(secrets.get(0).value, "TLS server CA cert", secretId); + if (certs.length == 0) { + throw new FetchErrors.InvalidResponseException( + "no PEM certificates found in TLS server CA cert secret \"" + secretId + "\""); + } + return certs; + }); + } + + private static X509Certificate[] parseCerts(byte[] secret, String what, String secretId) + throws FetchErrors.InvalidResponseException { + Matcher m = CERT_PEM_PATTERN.matcher(new String(secret, StandardCharsets.UTF_8)); + List certs = new ArrayList<>(); + try { + CertificateFactory factory = CertificateFactory.getInstance("X.509"); + while (m.find()) { + certs.add( + (X509Certificate) + factory.generateCertificate( + new ByteArrayInputStream(m.group().getBytes(StandardCharsets.US_ASCII)))); + } + } catch (GeneralSecurityException e) { + throw new FetchErrors.InvalidResponseException( + "invalid PEM certificate in " + what + " secret \"" + secretId + "\": " + e.getMessage(), + e); + } + return certs.toArray(new X509Certificate[0]); + } + + private static PrivateKey parseKey(byte[] secret, String keyId) + throws FetchErrors.InvalidResponseException { + Matcher m = KEY_PEM_PATTERN.matcher(new String(secret, StandardCharsets.UTF_8)); + if (!m.find()) { + throw new FetchErrors.InvalidResponseException( + "no PEM private key found in TLS client key secret \"" + keyId + "\""); + } + if (!m.group(1).equals("PRIVATE KEY")) { + throw new FetchErrors.InvalidResponseException( + "TLS client key secret \"" + + keyId + + "\" is not a PKCS#8 key: its PEM header is \"-----BEGIN " + + m.group(1) + + "-----\" instead of \"-----BEGIN PRIVATE KEY-----\". Convert it with: openssl " + + "pkcs8 -topk8 -nocrypt -in key.pem -out key-pkcs8.pem"); + } + byte[] der; + try { + der = Base64.getMimeDecoder().decode(m.group(2)); + } catch (IllegalArgumentException e) { + throw new FetchErrors.InvalidResponseException( + "invalid PEM private key in TLS client key secret \"" + keyId + "\"", e); + } + PKCS8EncodedKeySpec spec = new PKCS8EncodedKeySpec(der); + for (String algorithm : KEY_ALGORITHMS) { + try { + return KeyFactory.getInstance(algorithm).generatePrivate(spec); + } catch (GeneralSecurityException e) { + // Try the next algorithm + } + } + throw new FetchErrors.InvalidResponseException( + "invalid PEM private key in TLS client key secret \"" + keyId + "\""); + } + + /** Whether a private key matches a cert's public key, checked by signing and verifying. */ + private static boolean keyMatchesCert(PrivateKey key, X509Certificate cert) { + String algorithm; + switch (key.getAlgorithm()) { + case "RSA": + algorithm = "SHA256withRSA"; + break; + case "EC": + algorithm = "SHA256withECDSA"; + break; + case "DSA": + algorithm = "SHA256withDSA"; + break; + default: + return false; + } + byte[] data = "temporal-aws-secrets-manager".getBytes(StandardCharsets.US_ASCII); + try { + Signature signer = Signature.getInstance(algorithm); + signer.initSign(key); + signer.update(data); + byte[] signature = signer.sign(); + Signature verifier = Signature.getInstance(algorithm); + verifier.initVerify(cert.getPublicKey()); + verifier.update(data); + return verifier.verify(signature); + } catch (GeneralSecurityException e) { + // e.g. an EC key with an RSA cert + return false; + } + } +} diff --git a/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/SecretsManagerExtension.java b/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/SecretsManagerExtension.java new file mode 100644 index 0000000000..aab5739a68 --- /dev/null +++ b/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/SecretsManagerExtension.java @@ -0,0 +1,352 @@ +package io.temporal.aws.secretsmanager; + +import io.temporal.common.Experimental; +import io.temporal.envconfig.ClientConfigExtension; +import io.temporal.envconfig.ClientConfigExtensionRequest; +import io.temporal.envconfig.ClientConfigProfile; +import io.temporal.envconfig.ClientConfigTLS; +import io.temporal.serviceclient.WorkflowServiceStubsPlugin; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.regex.Pattern; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient; + +/** + * Client config extension that loads client credentials (API key and/or mTLS material) from AWS + * Secrets Manager. + * + *

Pass to {@link + * io.temporal.envconfig.LoadClientConfigProfileOptions.Builder#setExtensions(List)}: + * + *

{@code
+ * ClientConfigProfile profile =
+ *     ClientConfigProfile.load(
+ *         LoadClientConfigProfileOptions.newBuilder()
+ *             .setExtensions(Collections.singletonList(SecretsManagerExtension.getDefaultInstance()))
+ *             .build());
+ * WorkflowServiceStubs service =
+ *     WorkflowServiceStubs.newServiceStubs(profile.toWorkflowServiceStubsOptions());
+ * }
+ * + *

Secret IDs are configured in the {@code profile..ext.aws_secrets_manager} TOML table + * or with {@code TEMPORAL_EXT_AWS_SECRETS_MANAGER_*} environment variables. See the module README + * for details. + */ +@Experimental +public final class SecretsManagerExtension implements ClientConfigExtension { + /** The name of the extension, which is also its table key under {@code profile..ext}. */ + public static final String EXTENSION_NAME = "aws_secrets_manager"; + + private static final String ENV_API_KEY_ID = "TEMPORAL_EXT_AWS_SECRETS_MANAGER_API_KEY_ID"; + private static final String ENV_CLIENT_CERT_ID = + "TEMPORAL_EXT_AWS_SECRETS_MANAGER_TLS_CLIENT_CERT_ID"; + private static final String ENV_CLIENT_KEY_ID = + "TEMPORAL_EXT_AWS_SECRETS_MANAGER_TLS_CLIENT_KEY_ID"; + private static final String ENV_SERVER_CA_CERT_ID = + "TEMPORAL_EXT_AWS_SECRETS_MANAGER_TLS_SERVER_CA_CERT_ID"; + + // Same env var and default as the AWS Parameters and Secrets Lambda Extension's cache TTL + private static final String ENV_SECRETS_MANAGER_TTL = "SECRETS_MANAGER_TTL"; + private static final Pattern WHOLE_NUMBER = Pattern.compile("[0-9]+"); + private static final Duration DEFAULT_REFRESH_INTERVAL = Duration.ofMinutes(5); + private static final Duration DEFAULT_MAX_STALENESS = Duration.ofHours(24); + + private static final SecretsManagerExtension DEFAULT_INSTANCE = newBuilder().build(); + + /** Returns an extension with the default options. */ + public static SecretsManagerExtension getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + /** Creates a builder for an extension with non-default options. */ + public static Builder newBuilder() { + return new Builder(); + } + + // A SecretsManagerClient, held as an Object so that this class can be loaded without the AWS SDK + @Nullable private final Object secretsClient; + @Nullable private final Duration refreshInterval; + @Nullable private final Duration maxStaleness; + private final Map processEnv; + + private SecretsManagerExtension(Builder builder) { + this.secretsClient = builder.secretsClient; + this.refreshInterval = builder.refreshInterval; + this.maxStaleness = builder.maxStaleness; + this.processEnv = builder.processEnv; + } + + /** The extension name, {@code aws_secrets_manager}. */ + @Nonnull + @Override + public String getName() { + return EXTENSION_NAME; + } + + /** + * Reads secret IDs from the profile and env, and returns a plugin that fetches the configured + * secrets, or null if none are configured. + */ + @Nullable + @Override + public WorkflowServiceStubsPlugin getClientPlugin(@Nonnull ClientConfigExtensionRequest request) { + ClientConfigProfile profile = request.getProfile(); + SecretCache.SecretIds ids = loadSecretIds(profile, request.getEnv()); + if (ids.isEmpty()) { + return null; + } + + Duration refresh = refreshInterval(); + Duration staleness = maxStaleness; + if (staleness == null || staleness.isZero() || staleness.isNegative()) { + staleness = DEFAULT_MAX_STALENESS; + } + if (staleness.compareTo(refresh) < 0) { + staleness = refresh; + } + + Fetcher fetcher = + secretsClient != null + ? SdkFetcher.create(secretsClient) + : Fetcher.newDefaultFetcher(processEnv); + SecretCache cache = new SecretCache(fetcher, ids, nanos(refresh), nanos(staleness)); + ClientConfigTLS tls = profile.getTls(); + return new SecretsManagerPlugin( + ids, cache, tls != null && !Boolean.TRUE.equals(tls.isDisabled()) ? tls : null); + } + + private Duration refreshInterval() { + if (refreshInterval != null) { + if (refreshInterval.isNegative()) { + throw new IllegalArgumentException("refreshInterval must not be negative"); + } + return refreshInterval; + } + // Read from the process env rather than the config env, like + // PARAMETERS_SECRETS_EXTENSION_HTTP_PORT, since this is Lambda runtime config shared with the + // Lambda extension rather than Temporal client config + String ttl = processEnv.get(ENV_SECRETS_MANAGER_TTL); + if (ttl == null || ttl.isEmpty()) { + return DEFAULT_REFRESH_INTERVAL; + } + if (WHOLE_NUMBER.matcher(ttl).matches()) { + try { + return Duration.ofSeconds(Long.parseLong(ttl)); + } catch (NumberFormatException e) { + // Too large, handled below + } + } + throw new IllegalArgumentException( + "invalid " + + ENV_SECRETS_MANAGER_TTL + + " \"" + + ttl + + "\", must be a non-negative whole number of seconds"); + } + + /** Nanoseconds, saturating rather than overflowing. */ + private static long nanos(Duration d) { + try { + return d.toNanos(); + } catch (ArithmeticException e) { + return Long.MAX_VALUE; + } + } + + private static SecretCache.SecretIds loadSecretIds( + ClientConfigProfile profile, @Nullable Map env) { + Map fields = new HashMap<>(); + applyToml(fields, profile.getExt().get(EXTENSION_NAME)); + if (env != null) { + // Like envconfig, set-but-empty env vars override (and so can unset) file values + putEnv(fields, env, ENV_API_KEY_ID, "api_key"); + putEnv(fields, env, ENV_CLIENT_CERT_ID, "client_cert"); + putEnv(fields, env, ENV_CLIENT_KEY_ID, "client_key"); + putEnv(fields, env, ENV_SERVER_CA_CERT_ID, "server_ca_cert"); + } + SecretCache.SecretIds ids = + new SecretCache.SecretIds( + fields.getOrDefault("api_key", ""), + fields.getOrDefault("client_cert", ""), + fields.getOrDefault("client_key", ""), + fields.getOrDefault("server_ca_cert", "")); + validate(ids, profile); + return ids; + } + + private static void putEnv( + Map fields, Map env, String name, String field) { + String value = env.get(name); + if (value != null) { + fields.put(field, value); + } + } + + private static void applyToml(Map fields, @Nullable Map table) { + if (table == null) { + return; + } + for (Map.Entry e : table.entrySet()) { + String key = e.getKey(); + if (key.equals("api_key_id")) { + fields.put("api_key", tomlString(key, e.getValue())); + } else if (key.equals("tls")) { + if (!(e.getValue() instanceof Map)) { + throw new IllegalArgumentException( + "ext." + EXTENSION_NAME + ".tls must be a table, got " + typeName(e.getValue())); + } + for (Map.Entry tlsEntry : ((Map) e.getValue()).entrySet()) { + String tlsKey = String.valueOf(tlsEntry.getKey()); + String field; + switch (tlsKey) { + case "client_cert_id": + field = "client_cert"; + break; + case "client_key_id": + field = "client_key"; + break; + case "server_ca_cert_id": + field = "server_ca_cert"; + break; + default: + throw new IllegalArgumentException( + "unknown key \"" + tlsKey + "\" in ext." + EXTENSION_NAME + ".tls"); + } + fields.put(field, tomlString("tls." + tlsKey, tlsEntry.getValue())); + } + } else { + throw new IllegalArgumentException("unknown key \"" + key + "\" in ext." + EXTENSION_NAME); + } + } + } + + private static String tomlString(String key, Object value) { + if (!(value instanceof String)) { + throw new IllegalArgumentException( + "ext." + EXTENSION_NAME + "." + key + " must be a string, got " + typeName(value)); + } + return (String) value; + } + + private static String typeName(@Nullable Object value) { + if (value == null) { + return "null"; + } + if (value instanceof List) { + return "array"; + } + if (value instanceof Map) { + return "table"; + } + if (value instanceof Number) { + return "number"; + } + if (value instanceof Boolean) { + return "boolean"; + } + if (value instanceof String) { + return "string"; + } + return value.getClass().getSimpleName(); + } + + private static void validate(SecretCache.SecretIds ids, ClientConfigProfile profile) { + List conflicts = new ArrayList<>(); + if (!ids.apiKey.isEmpty() && !isEmpty(profile.getApiKey())) { + conflicts.add("API key secret ID cannot be set with an API key"); + } + ClientConfigTLS tls = profile.getTls(); + if (tls != null) { + if (!ids.clientCert.isEmpty() + && (!isEmpty(tls.getClientCertPath()) || !isEmpty(tls.getClientCertData()))) { + conflicts.add("TLS client cert secret ID cannot be set with client cert path or data"); + } + if (!ids.clientKey.isEmpty() + && (!isEmpty(tls.getClientKeyPath()) || !isEmpty(tls.getClientKeyData()))) { + conflicts.add("TLS client key secret ID cannot be set with client key path or data"); + } + if (!ids.serverCaCert.isEmpty() + && (!isEmpty(tls.getServerCACertPath()) || !isEmpty(tls.getServerCACertData()))) { + conflicts.add( + "TLS server CA cert secret ID cannot be set with server CA cert path or data"); + } + if (ids.hasTls() && Boolean.TRUE.equals(tls.isDisabled())) { + conflicts.add("TLS secret IDs cannot be set when TLS is disabled"); + } + } + if (ids.clientCert.isEmpty() != ids.clientKey.isEmpty()) { + conflicts.add("TLS client cert and key secret IDs must be set together"); + } + if (!conflicts.isEmpty()) { + Collections.sort(conflicts); + throw new IllegalArgumentException("invalid config: " + String.join("; ", conflicts)); + } + } + + private static boolean isEmpty(@Nullable String s) { + return s == null || s.isEmpty(); + } + + private static boolean isEmpty(@Nullable byte[] b) { + return b == null || b.length == 0; + } + + /** Builder for {@link SecretsManagerExtension}. */ + public static final class Builder { + @Nullable private Object secretsClient; + @Nullable private Duration refreshInterval; + @Nullable private Duration maxStaleness; + private Map processEnv = System.getenv(); + + private Builder() {} + + /** + * Sets the Secrets Manager client used to fetch secrets. If unset, secrets are fetched through + * the AWS Parameters and Secrets Lambda Extension if it is detected, and otherwise with a + * client created from the default AWS config, which requires {@code + * software.amazon.awssdk:secretsmanager} on the classpath. + */ + public Builder setSecretsClient(@Nullable SecretsManagerClient secretsClient) { + this.secretsClient = secretsClient; + return this; + } + + /** + * Sets how long a fetched secret is used before it is refreshed. If unset, defaults to the + * {@code SECRETS_MANAGER_TTL} env var (non-negative whole seconds, where 0 refreshes on every + * use) as used by the AWS Parameters and Secrets Lambda Extension, or 5 minutes if that is + * unset. + */ + public Builder setRefreshInterval(@Nullable Duration refreshInterval) { + this.refreshInterval = refreshInterval; + return this; + } + + /** + * Sets how long after its last successful fetch a secret may still be used if refreshing it + * fails. Defaults to 24 hours. Values less than the refresh interval are treated as the refresh + * interval. + */ + public Builder setMaxStaleness(@Nullable Duration maxStaleness) { + this.maxStaleness = maxStaleness; + return this; + } + + /** Overrides the process env, for tests. */ + Builder setProcessEnv(Map processEnv) { + this.processEnv = Objects.requireNonNull(processEnv); + return this; + } + + public SecretsManagerExtension build() { + return new SecretsManagerExtension(this); + } + } +} diff --git a/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/SecretsManagerPlugin.java b/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/SecretsManagerPlugin.java new file mode 100644 index 0000000000..c23b0640ae --- /dev/null +++ b/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/SecretsManagerPlugin.java @@ -0,0 +1,163 @@ +package io.temporal.aws.secretsmanager; + +import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts; +import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext; +import io.grpc.netty.shaded.io.netty.handler.ssl.SslContextBuilder; +import io.grpc.netty.shaded.io.netty.handler.ssl.util.InsecureTrustManagerFactory; +import io.temporal.authorization.AuthorizationGrpcMetadataProvider; +import io.temporal.envconfig.ClientConfigTLS; +import io.temporal.serviceclient.GrpcMetadataProvider; +import io.temporal.serviceclient.WorkflowServiceStubs; +import io.temporal.serviceclient.WorkflowServiceStubsOptions; +import io.temporal.serviceclient.WorkflowServiceStubsPlugin; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Collection; +import java.util.function.Supplier; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * Fetches the configured secrets when service stubs are created, and applies them to the stubs' + * options. + */ +final class SecretsManagerPlugin implements WorkflowServiceStubsPlugin { + static final String NAME = "io.temporal.aws.secretsmanager"; + + final SecretCache.SecretIds ids; + final SecretCache cache; + + /** + * The profile's TLS config if TLS isn't disabled, for which envconfig created an SSL context that + * this plugin replaces, or null. + */ + @Nullable private final ClientConfigTLS profileTls; + + /** + * The API key is refreshed at most this often, and retried this long after a failed refresh. + * Checking more often would only be cache hits, and with a refresh interval of 0 (no caching), + * this bounds the fetch rate. Overridable in tests. + */ + volatile long minRefreshIntervalNanos = SecretCache.FAILED_REFRESH_RETRY_INTERVAL_NANOS; + + /** + * Telemetry for the final options of the most recently created stubs. Read when used rather than + * captured, since the final options are only seen in {@link #connectServiceClient}, after {@link + * #configureServiceStubs} installs the API key supplier. + */ + private volatile Telemetry telemetry = new Telemetry(null); + + SecretsManagerPlugin( + SecretCache.SecretIds ids, SecretCache cache, @Nullable ClientConfigTLS profileTls) { + this.ids = ids; + this.cache = cache; + this.profileTls = profileTls; + } + + @Nonnull + @Override + public String getName() { + return NAME; + } + + @Override + public void configureServiceStubs(@Nonnull WorkflowServiceStubsOptions.Builder builder) { + WorkflowServiceStubsOptions current = builder.build(); + + SecretCache.CacheEntry apiKey = cache.apiKey; + if (apiKey != null) { + Collection providers = current.getGrpcMetadataProviders(); + if (providers != null) { + for (GrpcMetadataProvider provider : providers) { + if (provider instanceof AuthorizationGrpcMetadataProvider) { + throw new IllegalStateException( + "cannot use API key from AWS Secrets Manager, API key already set"); + } + } + } + // Called for each request, so a refreshed API key is used without waiting for the refresh. + // Like an API key from envconfig, this enables TLS unless it's explicitly configured. + builder.addApiKey(() -> cache.getNow(apiKey, minRefreshIntervalNanos, telemetry)); + } + + if (ids.hasTls()) { + if (profileTls == null && current.getSslContext() != null) { + throw new IllegalStateException( + "cannot use TLS config from AWS Secrets Manager, SSL context already set"); + } + // The SSL context must be set on the options, so these fetches can't wait for + // connectServiceClient and only report to a metrics scope that's already set + builder.setSslContext(sslContext(new Telemetry(current.getMetricsScope()))); + } + } + + @Nonnull + @Override + public WorkflowServiceStubs connectServiceClient( + @Nonnull WorkflowServiceStubsOptions options, @Nonnull Supplier next) { + // Unlike in configureServiceStubs, these are the final options, after all plugins have + // configured them and defaults are applied + Telemetry tel = new Telemetry(options.getMetricsScope()); + telemetry = tel; + // Fetched now so that creating the stubs fails on a bad secret instead of the first call + if (cache.apiKey != null) { + cache.get(cache.apiKey, tel); + } + return next.get(); + } + + /** + * Creates an SSL context from the secrets and the profile's other TLS settings, the same way + * envconfig does from the profile alone. + */ + private SslContext sslContext(Telemetry tel) { + SslContextBuilder builder = GrpcSslContexts.forClient(); + try { + if (cache.serverCaCert != null) { + builder.trustManager(cache.get(cache.serverCaCert, tel)); + } else { + try (InputStream ca = + profileTls != null + ? open(profileTls.getServerCACertPath(), profileTls.getServerCACertData()) + : null) { + if (ca != null) { + builder.trustManager(ca); + } else if (profileTls != null + && Boolean.TRUE.equals(profileTls.isDisableHostVerification())) { + builder.trustManager(InsecureTrustManagerFactory.INSTANCE); + } + } + } + + if (cache.clientCert != null) { + SecretCache.ClientCertPair pair = cache.get(cache.clientCert, tel); + builder.keyManager(pair.key, pair.chain); + } else if (profileTls != null) { + try (InputStream cert = + open(profileTls.getClientCertPath(), profileTls.getClientCertData()); + InputStream key = open(profileTls.getClientKeyPath(), profileTls.getClientKeyData())) { + if (cert != null && key != null) { + builder.keyManager(cert, key); + } + } + } + return builder.build(); + } catch (IOException e) { + throw new RuntimeException("Unable to create SSL context", e); + } + } + + @Nullable + private static InputStream open(@Nullable String path, @Nullable byte[] data) throws IOException { + if (path != null && !path.isEmpty()) { + return Files.newInputStream(Paths.get(path)); + } + if (data != null && data.length > 0) { + return new ByteArrayInputStream(data); + } + return null; + } +} diff --git a/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/Telemetry.java b/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/Telemetry.java new file mode 100644 index 0000000000..de05e5a94f --- /dev/null +++ b/contrib/temporal-aws-secrets-manager/src/main/java/io/temporal/aws/secretsmanager/Telemetry.java @@ -0,0 +1,109 @@ +package io.temporal.aws.secretsmanager; + +import com.uber.m3.tally.NoopScope; +import com.uber.m3.tally.Scope; +import java.util.HashMap; +import java.util.Map; +import javax.annotation.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Metrics for a client's metric scope. Metric and label names must be the same across SDKs. */ +class Telemetry { + private static final Logger log = LoggerFactory.getLogger(Telemetry.class); + + static final String METRIC_FETCH_FAILED = "temporal_aws_secrets_manager_fetch_failed"; + static final String METRIC_FETCH_LATENCY = "temporal_aws_secrets_manager_fetch_latency"; + static final String METRIC_CACHE_LOOKUP = "temporal_aws_secrets_manager_cache_lookup"; + + static final String LABEL_SECRET_TYPE = "secret_type"; + static final String LABEL_FETCH_PATH = "fetch_path"; + static final String LABEL_FAILURE_REASON = "failure_reason"; + static final String LABEL_FALLBACK = "fallback"; + static final String LABEL_OUTCOME = "outcome"; + static final String LABEL_RESULT = "result"; + + // secret_type values. Client cert and key secrets are fetched together and reported as + // tls_client_cert in metrics. + static final String SECRET_TYPE_API_KEY = "api_key"; + static final String SECRET_TYPE_TLS_CLIENT_CERT = "tls_client_cert"; + static final String SECRET_TYPE_TLS_CLIENT_KEY = "tls_client_key"; + static final String SECRET_TYPE_TLS_SERVER_CA_CERT = "tls_server_ca_cert"; + + // fetch_path values: the Lambda extension, or a direct fetch with the AWS SDK + static final String FETCH_PATH_EXTENSION = "extension"; + static final String FETCH_PATH_SDK = "sdk"; + + // AWS error codes mapped to failure_reason values. Other codes are "other". + private static final Map AWS_ERROR_CODE_FAILURE_REASONS = new HashMap<>(); + + static { + AWS_ERROR_CODE_FAILURE_REASONS.put("ResourceNotFoundException", "resource_not_found"); + AWS_ERROR_CODE_FAILURE_REASONS.put("AccessDeniedException", "access_denied"); + AWS_ERROR_CODE_FAILURE_REASONS.put("DecryptionFailure", "decryption_failure"); + AWS_ERROR_CODE_FAILURE_REASONS.put("InvalidRequestException", "invalid_request"); + AWS_ERROR_CODE_FAILURE_REASONS.put("InvalidParameterException", "invalid_request"); + AWS_ERROR_CODE_FAILURE_REASONS.put("ThrottlingException", "throttling"); + AWS_ERROR_CODE_FAILURE_REASONS.put("InternalServiceError", "internal_service_error"); + } + + private final Scope scope; + + Telemetry(@Nullable Scope scope) { + this.scope = scope != null ? scope : new NoopScope(); + } + + /** Returns the failure_reason metric label for a fetch error. */ + static String classifyFailure(Throwable err) { + if (err instanceof FetchErrors.InvalidResponseException) { + return "invalid_response"; + } + if (err instanceof FetchErrors.LambdaExtensionStatusException) { + int status = ((FetchErrors.LambdaExtensionStatusException) err).status; + if (status >= 400 && status < 500) { + return "client_error"; + } + if (status >= 500 && status < 600) { + return "server_error"; + } + return "other"; + } + if (err instanceof FetchErrors.FetchTimeoutException) { + return "timeout"; + } + if (err instanceof FetchErrors.AwsErrorException) { + String code = ((FetchErrors.AwsErrorException) err).errorCode; + String reason = code != null ? AWS_ERROR_CODE_FAILURE_REASONS.get(code) : null; + return reason != null ? reason : "other"; + } + // The Lambda extension doesn't return the underlying AWS error, only an HTTP status with a + // generic message, so its errors are only classified by status class + return "other"; + } + + void cacheLookup(Map tags) { + record(() -> scope.tagged(tags).counter(METRIC_CACHE_LOOKUP).inc(1)); + } + + void fetchFailed(Map tags) { + record(() -> scope.tagged(tags).counter(METRIC_FETCH_FAILED).inc(1)); + } + + void fetchLatency(Map tags, long nanos) { + record( + () -> + scope + .tagged(tags) + .timer(METRIC_FETCH_LATENCY) + .record(com.uber.m3.util.Duration.ofNanos(nanos))); + } + + /** Records a metric, never throwing, since recording failures shouldn't affect fetches. */ + private static void record(Runnable fn) { + try { + fn.run(); + } catch (RuntimeException e) { + log.warn("Failed recording AWS Secrets Manager metric", e); + } + } +} diff --git a/contrib/temporal-aws-secrets-manager/src/test/java/io/temporal/aws/secretsmanager/FetcherTest.java b/contrib/temporal-aws-secrets-manager/src/test/java/io/temporal/aws/secretsmanager/FetcherTest.java new file mode 100644 index 0000000000..e29765cd01 --- /dev/null +++ b/contrib/temporal-aws-secrets-manager/src/test/java/io/temporal/aws/secretsmanager/FetcherTest.java @@ -0,0 +1,249 @@ +package io.temporal.aws.secretsmanager; + +import static org.junit.Assert.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.sun.net.httpserver.HttpServer; +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import software.amazon.awssdk.awscore.exception.AwsErrorDetails; +import software.amazon.awssdk.core.SdkBytes; +import software.amazon.awssdk.core.exception.ApiCallTimeoutException; +import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient; +import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueRequest; +import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueResponse; +import software.amazon.awssdk.services.secretsmanager.model.SecretsManagerException; + +public class FetcherTest { + private HttpServer server; + private int port; + private final List requests = new ArrayList<>(); + private volatile Function respond = id -> new String[] {"404", "not found"}; + + @Before + public void setUp() throws Exception { + server = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0); + server.createContext( + "/", + exchange -> { + String query = exchange.getRequestURI().getRawQuery(); + synchronized (requests) { + requests.add( + exchange.getRequestURI().getRawPath() + + "?" + + query + + " token=" + + exchange.getRequestHeaders().getFirst("X-Aws-Parameters-Secrets-Token")); + } + String id = java.net.URLDecoder.decode(query.substring("secretId=".length()), "UTF-8"); + String[] resp = respond.apply(id); + byte[] body = resp[1].getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(Integer.parseInt(resp[0]), body.length); + try (OutputStream out = exchange.getResponseBody()) { + out.write(body); + } + }); + server.start(); + port = server.getAddress().getPort(); + } + + @After + public void tearDown() { + server.stop(0); + } + + private LambdaExtensionFetcher extensionFetcher() { + return new LambdaExtensionFetcher( + String.valueOf(port), Collections.singletonMap("AWS_SESSION_TOKEN", "token")); + } + + @Test + public void lambdaExtensionFetcher() throws Exception { + respond = + id -> + id.equals("binary/secret") + ? new String[] { + "200", + "{\"SecretBinary\": \"" + + Base64.getEncoder().encodeToString(new byte[] {1, 2, 3}) + + "\", \"VersionId\": \"v2\"}" + } + : new String[] {"200", "{\"SecretString\": \"value\", \"VersionId\": \"v1\"}"}; + LambdaExtensionFetcher fetcher = extensionFetcher(); + assertEquals("extension", fetcher.path()); + Fetcher.SecretValue string = fetcher.fetch("string"); + assertEquals("value", new String(string.value, StandardCharsets.UTF_8)); + assertEquals("v1", string.versionId); + Fetcher.SecretValue binary = fetcher.fetch("binary/secret"); + assertArrayEquals(new byte[] {1, 2, 3}, binary.value); + assertEquals("v2", binary.versionId); + assertEquals( + new ArrayList<>( + java.util.Arrays.asList( + "/secretsmanager/get?secretId=string token=token", + "/secretsmanager/get?secretId=binary%2Fsecret token=token")), + requests); + } + + @Test + public void lambdaExtensionFetcherErrors() { + Object[][] cases = { + {"400", "bad", "client_error"}, + {"500", "oops", "server_error"}, + {"200", "not json", "invalid_response"}, + {"200", "\"string\"", "invalid_response"}, + {"200", "{\"SecretString\": 1}", "invalid_response"}, + {"200", "{\"VersionId\": \"v1\"}", "invalid_response"}, + }; + for (Object[] c : cases) { + respond = id -> new String[] {(String) c[0], (String) c[1]}; + Exception e = assertThrows(Exception.class, () -> extensionFetcher().fetch("id")); + assertEquals(c[1] + ": " + e, c[2], Telemetry.classifyFailure(e)); + } + } + + @Test + public void lambdaExtensionFetcherConnectionRefused() throws Exception { + int closedPort; + try (ServerSocket socket = new ServerSocket(0, 0, InetAddress.getLoopbackAddress())) { + closedPort = socket.getLocalPort(); + } + Exception e = + assertThrows( + Exception.class, + () -> + new LambdaExtensionFetcher(String.valueOf(closedPort), Collections.emptyMap()) + .fetch("id")); + assertEquals("other", Telemetry.classifyFailure(e)); + } + + @Test + public void sdkFetcher() throws Exception { + SecretsManagerClient client = mock(SecretsManagerClient.class); + List reqs = new ArrayList<>(); + when(client.getSecretValue(any(GetSecretValueRequest.class))) + .thenAnswer( + inv -> { + GetSecretValueRequest req = inv.getArgument(0); + reqs.add(req); + return req.secretId().equals("binary") + ? GetSecretValueResponse.builder() + .secretBinary(SdkBytes.fromByteArray(new byte[] {1, 2})) + .versionId("v2") + .build() + : GetSecretValueResponse.builder().secretString("value").versionId("v1").build(); + }); + Fetcher fetcher = SdkFetcher.create(client); + assertEquals("sdk", fetcher.path()); + assertEquals("value", new String(fetcher.fetch("string").value, StandardCharsets.UTF_8)); + assertArrayEquals(new byte[] {1, 2}, fetcher.fetch("binary").value); + // Each request is bounded by the fetch timeout + assertTrue(reqs.get(0).overrideConfiguration().get().apiCallTimeout().isPresent()); + } + + @Test + public void sdkFetcherErrors() { + SecretsManagerClient client = mock(SecretsManagerClient.class); + when(client.getSecretValue(any(GetSecretValueRequest.class))) + .thenThrow( + SecretsManagerException.builder() + .message("not found") + .awsErrorDetails( + AwsErrorDetails.builder().errorCode("ResourceNotFoundException").build()) + .build()) + .thenThrow(ApiCallTimeoutException.create(10_000)) + .thenReturn(GetSecretValueResponse.builder().versionId("v1").build()); + Fetcher fetcher = SdkFetcher.create(client); + assertEquals( + "resource_not_found", + Telemetry.classifyFailure(assertThrows(Exception.class, () -> fetcher.fetch("id")))); + assertEquals( + "timeout", + Telemetry.classifyFailure(assertThrows(Exception.class, () -> fetcher.fetch("id")))); + Exception e = assertThrows(Exception.class, () -> fetcher.fetch("id")); + assertEquals("invalid_response", Telemetry.classifyFailure(e)); + assertEquals("secret has no value", e.getMessage()); + } + + @Test + public void classifyAwsErrors() { + Map cases = new HashMap<>(); + cases.put("ResourceNotFoundException", "resource_not_found"); + cases.put("AccessDeniedException", "access_denied"); + cases.put("DecryptionFailure", "decryption_failure"); + cases.put("InvalidRequestException", "invalid_request"); + cases.put("InvalidParameterException", "invalid_request"); + cases.put("ThrottlingException", "throttling"); + cases.put("InternalServiceError", "internal_service_error"); + cases.put("SomethingElse", "other"); + for (Map.Entry c : cases.entrySet()) { + assertEquals( + c.getKey(), + c.getValue(), + Telemetry.classifyFailure( + new FetchErrors.AwsErrorException(c.getKey(), new Exception()))); + } + assertEquals( + "other", + Telemetry.classifyFailure(new FetchErrors.AwsErrorException(null, new Exception()))); + } + + @Test + public void classifyOtherErrors() { + assertEquals( + "invalid_response", + Telemetry.classifyFailure(new FetchErrors.InvalidResponseException("bad"))); + assertEquals( + "timeout", Telemetry.classifyFailure(new FetchErrors.FetchTimeoutException("slow", null))); + assertEquals( + "client_error", + Telemetry.classifyFailure(new FetchErrors.LambdaExtensionStatusException(403, ""))); + assertEquals( + "server_error", + Telemetry.classifyFailure(new FetchErrors.LambdaExtensionStatusException(503, ""))); + assertEquals( + "other", + Telemetry.classifyFailure(new FetchErrors.LambdaExtensionStatusException(302, ""))); + assertEquals("other", Telemetry.classifyFailure(new RuntimeException())); + } + + @Test + public void defaultFetcherSelection() { + Map env = new HashMap<>(); + env.put(Fetcher.ENV_LAMBDA_EXTENSION_PORT, "1234"); + assertTrue(Fetcher.newDefaultFetcher(env) instanceof LambdaExtensionFetcher); + + // Empty disables detection, even in Lambda + env.put(Fetcher.ENV_LAMBDA_EXTENSION_PORT, ""); + env.put(Fetcher.ENV_LAMBDA_FUNCTION_NAME, "fn"); + assertTrue(Fetcher.newDefaultFetcher(env) instanceof SdkFetcher); + + // Not probed outside Lambda + assertTrue(Fetcher.newDefaultFetcher(new HashMap<>()) instanceof SdkFetcher); + } + + @Test + public void probe() throws Exception { + assertTrue(Fetcher.probe(InetAddress.getLoopbackAddress().getHostAddress(), port)); + int closedPort; + try (ServerSocket socket = new ServerSocket(0, 0, InetAddress.getLoopbackAddress())) { + closedPort = socket.getLocalPort(); + } + assertFalse(Fetcher.probe(InetAddress.getLoopbackAddress().getHostAddress(), closedPort)); + } +} diff --git a/contrib/temporal-aws-secrets-manager/src/test/java/io/temporal/aws/secretsmanager/SecretCacheTest.java b/contrib/temporal-aws-secrets-manager/src/test/java/io/temporal/aws/secretsmanager/SecretCacheTest.java new file mode 100644 index 0000000000..944556bf31 --- /dev/null +++ b/contrib/temporal-aws-secrets-manager/src/test/java/io/temporal/aws/secretsmanager/SecretCacheTest.java @@ -0,0 +1,396 @@ +package io.temporal.aws.secretsmanager; + +import static io.temporal.aws.secretsmanager.TestHelpers.*; +import static org.junit.Assert.*; + +import io.temporal.aws.secretsmanager.SecretCache.ClientCertPair; +import io.temporal.aws.secretsmanager.SecretCache.SecretFetchException; +import io.temporal.aws.secretsmanager.SecretCache.SecretIds; +import java.security.cert.X509Certificate; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; +import org.junit.Test; + +public class SecretCacheTest { + private static final long REFRESH = TimeUnit.MINUTES.toNanos(1); + private static final long MAX_STALENESS = TimeUnit.HOURS.toNanos(1); + private static final long RETRY = SecretCache.FAILED_REFRESH_RETRY_INTERVAL_NANOS; + + private final FakeFetcher fetcher = new FakeFetcher(); + private final AtomicLong clock = new AtomicLong(1_000_000_000L); + private final RecordingTelemetry tel = new RecordingTelemetry(); + + private SecretCache newCache(SecretIds ids, long refresh) { + SecretCache cache = new SecretCache(fetcher, ids, refresh, MAX_STALENESS); + cache.now = clock::get; + return cache; + } + + private SecretCache newApiKeyCache(long refresh) { + return newCache(new SecretIds("key", "", "", ""), refresh); + } + + private static Map tags(String... kv) { + Map m = new HashMap<>(); + for (int i = 0; i < kv.length; i += 2) { + m.put(kv[i], kv[i + 1]); + } + return m; + } + + private List results() { + return tel.take(Telemetry.METRIC_CACHE_LOOKUP).stream() + .map(t -> t.get(Telemetry.LABEL_RESULT)) + .collect(Collectors.toList()); + } + + @Test + public void hitMissAndRotation() { + fetcher.set("key", "v1", "ver1"); + SecretCache cache = newApiKeyCache(REFRESH); + assertEquals("v1", cache.get(cache.apiKey, tel)); + assertEquals("v1", cache.get(cache.apiKey, tel)); + assertEquals(1, fetcher.calls("key")); + assertEquals( + Arrays.asList( + tags("secret_type", "api_key", "fetch_path", "sdk", "result", "miss"), + tags("secret_type", "api_key", "fetch_path", "sdk", "result", "hit")), + tel.take(Telemetry.METRIC_CACHE_LOOKUP)); + assertEquals( + Arrays.asList(tags("secret_type", "api_key", "fetch_path", "sdk", "outcome", "success")), + tel.take(Telemetry.METRIC_FETCH_LATENCY)); + + clock.addAndGet(REFRESH); + fetcher.set("key", "v2", "ver2"); + assertEquals("v2", cache.get(cache.apiKey, tel)); + assertEquals(2, fetcher.calls("key")); + } + + @Test + public void zeroRefreshIntervalAlwaysFetches() { + fetcher.set("key", "v1", "ver1"); + SecretCache cache = newApiKeyCache(0); + cache.get(cache.apiKey, tel); + cache.get(cache.apiKey, tel); + assertEquals(2, fetcher.calls("key")); + } + + @Test + public void failureFallsBackToStaleValueAndWaitsBeforeRetrying() { + fetcher.set("key", "v1", "ver1"); + SecretCache cache = newApiKeyCache(REFRESH); + cache.get(cache.apiKey, tel); + tel.take(Telemetry.METRIC_CACHE_LOOKUP); + + fetcher.fail( + "key", new FetchErrors.AwsErrorException("AccessDeniedException", new Exception())); + clock.addAndGet(REFRESH); + assertEquals("v1", cache.get(cache.apiKey, tel)); + assertEquals( + Arrays.asList( + tags( + "secret_type", + "api_key", + "fetch_path", + "sdk", + "failure_reason", + "access_denied", + "fallback", + "stale")), + tel.take(Telemetry.METRIC_FETCH_FAILED)); + + // Not retried until the retry interval has passed + clock.addAndGet(RETRY - 1); + assertEquals("v1", cache.get(cache.apiKey, tel)); + assertEquals(2, fetcher.calls("key")); + clock.addAndGet(1); + assertEquals("v1", cache.get(cache.apiKey, tel)); + assertEquals(3, fetcher.calls("key")); + assertEquals( + Arrays.asList("miss_with_fallback", "miss_with_fallback", "miss_with_fallback"), results()); + + fetcher.set("key", "v1", "ver1"); + clock.addAndGet(RETRY); + assertEquals("v1", cache.get(cache.apiKey, tel)); + assertNull(cache.apiKey.failedAt); + } + + @Test + public void failureWithoutPreviousValueThrows() { + fetcher.fail( + "key", + new FetchErrors.AwsErrorException("ResourceNotFoundException", new Exception("nope"))); + SecretCache cache = newApiKeyCache(REFRESH); + SecretFetchException e = + assertThrows(SecretFetchException.class, () -> cache.get(cache.apiKey, tel)); + assertEquals("failed fetching AWS Secrets Manager secret \"key\": nope", e.getMessage()); + assertEquals("resource_not_found", e.reason); + assertEquals( + "none", tel.take(Telemetry.METRIC_FETCH_FAILED).get(0).get(Telemetry.LABEL_FALLBACK)); + assertTrue(results().isEmpty()); + } + + @Test + public void staleValueIsNotUsedPastMaxStaleness() { + fetcher.set("key", "v1", "ver1"); + SecretCache cache = newApiKeyCache(REFRESH); + cache.get(cache.apiKey, tel); + fetcher.fail("key", new Exception("down")); + clock.addAndGet(MAX_STALENESS); + assertThrows(SecretFetchException.class, () -> cache.get(cache.apiKey, tel)); + } + + @Test + public void concurrentGetsShareAFetch() throws Exception { + fetcher.set("key", "v1", "ver1"); + fetcher.gate = new CountDownLatch(1); + SecretCache cache = newApiKeyCache(REFRESH); + ExecutorService pool = Executors.newFixedThreadPool(2); + try { + Future a = pool.submit(() -> cache.get(cache.apiKey, tel)); + Future b = pool.submit(() -> cache.get(cache.apiKey, tel)); + waitUntil(() -> fetcher.calls("key") == 1); + fetcher.gate.countDown(); + assertEquals("v1", a.get(5, TimeUnit.SECONDS)); + assertEquals("v1", b.get(5, TimeUnit.SECONDS)); + assertEquals(1, fetcher.calls("key")); + } finally { + pool.shutdownNow(); + } + } + + @Test + public void refreshingServesStaleValueToOtherCallers() throws Exception { + fetcher.set("key", "v1", "ver1"); + SecretCache cache = newApiKeyCache(REFRESH); + cache.get(cache.apiKey, tel); + + fetcher.set("key", "v2", "ver2"); + fetcher.gate = new CountDownLatch(1); + clock.addAndGet(REFRESH); + ExecutorService pool = Executors.newSingleThreadExecutor(); + try { + Future refreshing = pool.submit(() -> cache.get(cache.apiKey, tel)); + waitUntil(() -> fetcher.calls("key") == 2); + assertEquals("v1", cache.get(cache.apiKey, tel)); + fetcher.gate.countDown(); + assertEquals("v2", refreshing.get(5, TimeUnit.SECONDS)); + } finally { + pool.shutdownNow(); + } + } + + @Test + public void callerTimingOutDoesNotCancelFetch() throws Exception { + fetcher.set("key", "v1", "ver1"); + fetcher.gate = new CountDownLatch(1); + SecretCache cache = newApiKeyCache(REFRESH); + cache.fetchTimeoutNanos = TimeUnit.MILLISECONDS.toNanos(10); + SecretCache.FetchWaitTimeoutException e = + assertThrows( + SecretCache.FetchWaitTimeoutException.class, () -> cache.get(cache.apiKey, tel)); + assertEquals( + "timed out after 10ms waiting for AWS Secrets Manager secret fetch", e.getMessage()); + + fetcher.gate.countDown(); + waitUntil(() -> inFlight(cache) == null); + assertEquals("v1", cache.get(cache.apiKey, tel)); + assertEquals(1, fetcher.calls("key")); + } + + @Test + public void timedOutWaitFallsBackToStaleValue() throws Exception { + fetcher.set("key", "v1", "ver1"); + SecretCache cache = newApiKeyCache(0); + cache.get(cache.apiKey, tel); + tel.take(Telemetry.METRIC_CACHE_LOOKUP); + + fetcher.gate = new CountDownLatch(1); + cache.fetchTimeoutNanos = TimeUnit.MILLISECONDS.toNanos(10); + clock.addAndGet(1); + assertEquals("v1", cache.get(cache.apiKey, tel)); + assertEquals(Arrays.asList("miss_with_fallback"), results()); + fetcher.gate.countDown(); + waitUntil(() -> inFlight(cache) == null); + } + + @Test + public void failedFetchStartDoesNotWedgeCache() { + fetcher.set("key", "v1", "ver1"); + SecretCache cache = newApiKeyCache(REFRESH); + cache.executor = + r -> { + throw new RejectedExecutionException("no threads"); + }; + SecretFetchException e = + assertThrows(SecretFetchException.class, () -> cache.get(cache.apiKey, tel)); + assertEquals("failed starting AWS Secrets Manager secret fetch", e.getMessage()); + assertNull(inFlight(cache)); + + cache.executor = Runnable::run; + assertEquals("v1", cache.get(cache.apiKey, tel)); + } + + @Test + public void getNowRefreshesInBackgroundWhenDue() throws Exception { + fetcher.set("key", "v1", "ver1"); + SecretCache cache = newApiKeyCache(0); + cache.get(cache.apiKey, tel); + long min = TimeUnit.SECONDS.toNanos(1); + + // A zero refresh interval is bounded by the minimum + fetcher.set("key", "v2", "ver2"); + assertEquals("v1", cache.getNow(cache.apiKey, min, tel)); + assertNull(inFlight(cache)); + clock.addAndGet(min); + assertEquals("v1", cache.getNow(cache.apiKey, min, tel)); + waitUntil(() -> "v2".equals(cache.getNow(cache.apiKey, min, tel))); + assertEquals(2, fetcher.calls("key")); + + // A failed refresh keeps the current value, even past max staleness, and isn't retried until + // the retry interval has passed + fetcher.fail("key", new Exception("down")); + clock.addAndGet(MAX_STALENESS); + assertEquals("v2", cache.getNow(cache.apiKey, min, tel)); + waitUntil(() -> cache.apiKey.failedAt != null && inFlight(cache) == null); + assertEquals("v2", cache.getNow(cache.apiKey, min, tel)); + assertEquals(3, fetcher.calls("key")); + clock.addAndGet(RETRY); + cache.getNow(cache.apiKey, min, tel); + waitUntil(() -> fetcher.calls("key") == 4 && inFlight(cache) == null); + } + + @Test + public void bundledClientCertAndKey() { + fetcher.set("bundle", "junk\n" + TEST_CERT_A + TEST_KEY_A, "ver1"); + SecretCache cache = newCache(new SecretIds("", "bundle", "bundle", ""), REFRESH); + ClientCertPair pair = cache.get(cache.clientCert, tel); + assertEquals(1, pair.chain.length); + assertEquals( + "CN=aws-secrets-manager-test-a", pair.chain[0].getSubjectX500Principal().getName()); + assertEquals("EC", pair.key.getAlgorithm()); + assertEquals(1, fetcher.calls("bundle")); + } + + @Test + public void separateClientCertAndKey() { + fetcher.set("cert", TEST_CERT_A, "ver1"); + fetcher.set("key", TEST_KEY_A, "ver2"); + SecretCache cache = newCache(new SecretIds("", "cert", "key", ""), REFRESH); + cache.get(cache.clientCert, tel); + // Both are reported as the client cert in metrics + assertEquals( + Arrays.asList( + tags("secret_type", "tls_client_cert", "fetch_path", "sdk", "result", "miss")), + tel.take(Telemetry.METRIC_CACHE_LOOKUP)); + } + + @Test + public void mismatchedClientCertAndKeyKeepThePreviousPair() { + fetcher.set("cert", TEST_CERT_A, "ver1"); + fetcher.set("key", TEST_KEY_A, "ver1"); + SecretCache cache = newCache(new SecretIds("", "cert", "key", ""), REFRESH); + cache.get(cache.clientCert, tel); + + // Fetched partway through rotating the separate secrets + fetcher.set("cert", TEST_CERT_B, "ver2"); + clock.addAndGet(REFRESH); + assertEquals( + subject(cache.get(cache.clientCert, tel).chain[0]), "CN=aws-secrets-manager-test-a"); + assertEquals( + "invalid_response", + tel.take(Telemetry.METRIC_FETCH_FAILED).get(0).get(Telemetry.LABEL_FAILURE_REASON)); + + fetcher.set("key", TEST_KEY_B, "ver2"); + clock.addAndGet(RETRY); + assertEquals( + subject(cache.get(cache.clientCert, tel).chain[0]), "CN=aws-secrets-manager-test-b"); + } + + @Test + public void invalidClientCertAndKey() { + String[][] cases = { + {TEST_KEY_A, TEST_KEY_A, "no PEM certificate found in TLS client cert secret \"cert\""}, + {TEST_CERT_A, TEST_CERT_A, "no PEM private key found in TLS client key secret \"key\""}, + { + TEST_CERT_A, + TEST_KEY_B, + "TLS client cert and key from AWS Secrets Manager secrets \"cert\" and \"key\" don't match" + }, + { + TEST_CERT_A, + TEST_KEY_A_EC, + "TLS client key secret \"key\" is not a PKCS#8 key: its PEM header is \"-----BEGIN EC" + + " PRIVATE KEY-----\" instead of \"-----BEGIN PRIVATE KEY-----\". Convert it with:" + + " openssl pkcs8 -topk8 -nocrypt -in key.pem -out key-pkcs8.pem" + }, + { + "-----BEGIN CERTIFICATE-----\nbm9wZQ==\n-----END CERTIFICATE-----", + TEST_KEY_A, + null // Message from the JDK + }, + { + TEST_CERT_A, + "-----BEGIN PRIVATE KEY-----\nbm9wZQ==\n-----END PRIVATE KEY-----", + "invalid PEM private key in TLS client key secret \"key\"" + }, + }; + for (String[] c : cases) { + fetcher.set("cert", c[0], "ver1"); + fetcher.set("key", c[1], "ver1"); + SecretCache cache = newCache(new SecretIds("", "cert", "key", ""), REFRESH); + SecretFetchException e = + assertThrows(SecretFetchException.class, () -> cache.get(cache.clientCert, tel)); + if (c[2] != null) { + assertEquals(c[2], e.getMessage()); + } else { + assertTrue(e.getMessage(), e.getMessage().startsWith("invalid PEM certificate in")); + } + assertEquals("invalid_response", e.reason); + } + } + + @Test + public void invalidUtf8ApiKey() { + fetcher.set("key", new byte[] {(byte) 0xff, (byte) 0xfe}, "ver1"); + SecretCache cache = newApiKeyCache(REFRESH); + SecretFetchException e = + assertThrows(SecretFetchException.class, () -> cache.get(cache.apiKey, tel)); + assertEquals("API key secret is not valid UTF-8", e.getMessage()); + assertEquals("invalid_response", e.reason); + } + + @Test + public void serverCaCert() { + fetcher.set("ca", TEST_CERT_A + "\n" + TEST_CERT_B, "ver1"); + SecretCache cache = newCache(new SecretIds("", "", "", "ca"), REFRESH); + X509Certificate[] certs = cache.get(cache.serverCaCert, tel); + assertEquals(2, certs.length); + + fetcher.set("ca", "nothing here", "ver1"); + SecretCache cache2 = newCache(new SecretIds("", "", "", "ca"), REFRESH); + SecretFetchException e = + assertThrows(SecretFetchException.class, () -> cache2.get(cache2.serverCaCert, tel)); + assertEquals("no PEM certificates found in TLS server CA cert secret \"ca\"", e.getMessage()); + } + + private static String subject(X509Certificate cert) { + return cert.getSubjectX500Principal().getName(); + } + + private static Object inFlight(SecretCache cache) { + synchronized (cache.apiKey != null ? cache.apiKey : cache.clientCert) { + return cache.apiKey != null ? cache.apiKey.inFlight : cache.clientCert.inFlight; + } + } +} diff --git a/contrib/temporal-aws-secrets-manager/src/test/java/io/temporal/aws/secretsmanager/SecretsManagerExtensionTest.java b/contrib/temporal-aws-secrets-manager/src/test/java/io/temporal/aws/secretsmanager/SecretsManagerExtensionTest.java new file mode 100644 index 0000000000..b861373d14 --- /dev/null +++ b/contrib/temporal-aws-secrets-manager/src/test/java/io/temporal/aws/secretsmanager/SecretsManagerExtensionTest.java @@ -0,0 +1,499 @@ +package io.temporal.aws.secretsmanager; + +import static io.temporal.aws.secretsmanager.TestHelpers.*; +import static org.junit.Assert.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.RETURNS_DEEP_STUBS; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.uber.m3.tally.Scope; +import io.grpc.Metadata; +import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts; +import io.temporal.authorization.AuthorizationGrpcMetadataProvider; +import io.temporal.envconfig.ClientConfigProfile; +import io.temporal.envconfig.LoadClientConfigProfileOptions; +import io.temporal.serviceclient.GrpcMetadataProvider; +import io.temporal.serviceclient.WorkflowServiceStubs; +import io.temporal.serviceclient.WorkflowServiceStubsOptions; +import io.temporal.serviceclient.WorkflowServiceStubsPlugin; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import org.junit.Test; +import software.amazon.awssdk.awscore.exception.AwsErrorDetails; +import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient; +import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueRequest; +import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueResponse; +import software.amazon.awssdk.services.secretsmanager.model.SecretsManagerException; + +public class SecretsManagerExtensionTest { + private static final String PROFILE = "[profile.default]\naddress = \"localhost:7233\"\n"; + private static final String EXT = "[profile.default.ext.aws_secrets_manager]\n"; + private static final String EXT_TLS = "[profile.default.ext.aws_secrets_manager.tls]\n"; + private static final String API_KEY_PROFILE = PROFILE + EXT + "api_key_id = \"id\"\n"; + private static final String ALL_PROFILE = + PROFILE + + EXT + + "api_key_id = \"api-key\"\n" + + EXT_TLS + + "client_cert_id = \"bundle\"\n" + + "client_key_id = \"bundle\"\n" + + "server_ca_cert_id = \"ca\"\n"; + + /** A fake Secrets Manager client that returns {@code secrets}, counting fetches of each. */ + private static final class FakeSecrets { + final Map secrets = new ConcurrentHashMap<>(); + final Map calls = new ConcurrentHashMap<>(); + private final AtomicInteger version = new AtomicInteger(); + final SecretsManagerClient client = mock(SecretsManagerClient.class); + + FakeSecrets() { + when(client.getSecretValue(any(GetSecretValueRequest.class))) + .thenAnswer( + inv -> { + String id = inv.getArgument(0).secretId(); + calls.computeIfAbsent(id, k -> new AtomicInteger()).incrementAndGet(); + Object value = secrets.get(id); + if (value == null) { + throw SecretsManagerException.builder() + .message("no secret " + id) + .awsErrorDetails( + AwsErrorDetails.builder().errorCode("ResourceNotFoundException").build()) + .build(); + } + if (value instanceof RuntimeException) { + throw (RuntimeException) value; + } + return GetSecretValueResponse.builder() + .secretString((String) value) + .versionId("v" + version.incrementAndGet()) + .build(); + }); + } + + FakeSecrets put(String id, Object value) { + secrets.put(id, value); + return this; + } + + int calls(String id) { + AtomicInteger n = calls.get(id); + return n != null ? n.get() : 0; + } + } + + private static FakeSecrets allSecrets() { + return new FakeSecrets() + .put("api-key", "secret-key") + .put("bundle", TEST_CERT_A + TEST_KEY_A) + .put("ca", TEST_CERT_A); + } + + private static ClientConfigProfile load( + String toml, Map env, SecretsManagerExtension.Builder extension) + throws Exception { + return ClientConfigProfile.load( + LoadClientConfigProfileOptions.newBuilder() + .setConfigFileData(toml.getBytes(StandardCharsets.UTF_8)) + .setEnvOverrides(env) + .setExtensions(Collections.singletonList(extension.build())) + .build()); + } + + private static SecretsManagerExtension.Builder extension(FakeSecrets secrets) { + return SecretsManagerExtension.newBuilder() + .setSecretsClient(secrets.client) + .setProcessEnv(Collections.emptyMap()); + } + + private static SecretsManagerPlugin plugin(ClientConfigProfile profile) { + assertEquals(1, profile.getPlugins().size()); + return (SecretsManagerPlugin) profile.getPlugins().get(0); + } + + private static SecretsManagerPlugin load(String toml, FakeSecrets secrets) throws Exception { + return plugin(load(toml, Collections.emptyMap(), extension(secrets))); + } + + private static String ids(SecretsManagerPlugin plugin) { + SecretCache.SecretIds ids = plugin.ids; + return String.join(",", ids.apiKey, ids.clientCert, ids.clientKey, ids.serverCaCert); + } + + /** Applies the plugin to the profile's service stubs options, without creating stubs. */ + private static WorkflowServiceStubsOptions configure(ClientConfigProfile profile) { + WorkflowServiceStubsOptions.Builder builder = + WorkflowServiceStubsOptions.newBuilder(profile.toWorkflowServiceStubsOptions()); + plugin(profile).configureServiceStubs(builder); + WorkflowServiceStubsOptions options = builder.build(); + plugin(profile).connectServiceClient(options, () -> null); + return options; + } + + private static String authorization(WorkflowServiceStubsOptions options) { + List providers = + options.getGrpcMetadataProviders().stream() + .filter(p -> p instanceof AuthorizationGrpcMetadataProvider) + .collect(Collectors.toList()); + assertEquals(1, providers.size()); + return providers + .get(0) + .getMetadata() + .get(Metadata.Key.of("authorization", Metadata.ASCII_STRING_MARSHALLER)); + } + + @Test + public void noSecretIdsMeansNoPlugin() throws Exception { + assertTrue( + load(PROFILE, Collections.emptyMap(), extension(new FakeSecrets())).getPlugins().isEmpty()); + assertTrue( + load(PROFILE + EXT, Collections.emptyMap(), extension(new FakeSecrets())) + .getPlugins() + .isEmpty()); + } + + @Test + public void secretIdsAreLoadedFromTomlAndEnv() throws Exception { + String toml = + PROFILE + + EXT + + "api_key_id = \"toml-api-key\"\n" + + EXT_TLS + + "client_cert_id = \"toml-cert\"\n" + + "client_key_id = \"toml-key\"\n" + + "server_ca_cert_id = \"toml-ca\"\n"; + assertEquals("toml-api-key,toml-cert,toml-key,toml-ca", ids(load(toml, new FakeSecrets()))); + + // Env vars override the file, and set-but-empty env vars unset it + Map env = new HashMap<>(); + env.put("TEMPORAL_EXT_AWS_SECRETS_MANAGER_API_KEY_ID", "env-api-key"); + env.put("TEMPORAL_EXT_AWS_SECRETS_MANAGER_TLS_SERVER_CA_CERT_ID", ""); + assertEquals( + "env-api-key,toml-cert,toml-key,", + ids(plugin(load(toml, env, extension(new FakeSecrets()))))); + + // Env vars are ignored when env loading is disabled + ClientConfigProfile profile = + ClientConfigProfile.load( + LoadClientConfigProfileOptions.newBuilder() + .setConfigFileData(PROFILE.getBytes(StandardCharsets.UTF_8)) + .setEnvOverrides( + Collections.singletonMap("TEMPORAL_EXT_AWS_SECRETS_MANAGER_API_KEY_ID", "id")) + .setDisableEnv(true) + .setExtensions(Collections.singletonList(extension(new FakeSecrets()).build())) + .build()); + assertTrue(profile.getPlugins().isEmpty()); + } + + @Test + public void invalidConfig() { + String[][] cases = { + {EXT + "foo = \"x\"", "unknown key \"foo\" in ext.aws_secrets_manager"}, + {EXT_TLS + "foo = \"x\"", "unknown key \"foo\" in ext.aws_secrets_manager.tls"}, + {EXT + "tls = \"x\"", "ext.aws_secrets_manager.tls must be a table, got string"}, + {EXT + "api_key_id = 1", "ext.aws_secrets_manager.api_key_id must be a string, got number"}, + { + EXT_TLS + "client_cert_id = []", + "ext.aws_secrets_manager.tls.client_cert_id must be a string, got array" + }, + { + "api_key = \"key\"\n" + EXT + "api_key_id = \"id\"", + "invalid config: API key secret ID cannot be set with an API key" + }, + { + "[profile.default.tls]\nclient_cert_data = \"x\"\nclient_key_data = \"y\"\n" + + EXT_TLS + + "client_cert_id = \"c\"\nclient_key_id = \"k\"", + "invalid config: TLS client cert secret ID cannot be set with client cert path or data; " + + "TLS client key secret ID cannot be set with client key path or data" + }, + { + "[profile.default.tls]\nserver_ca_cert_data = \"x\"\n" + + EXT_TLS + + "server_ca_cert_id = \"ca\"", + "invalid config: TLS server CA cert secret ID cannot be set with server CA cert path or" + + " data" + }, + { + "[profile.default.tls]\ndisabled = true\n" + EXT_TLS + "server_ca_cert_id = \"ca\"", + "invalid config: TLS secret IDs cannot be set when TLS is disabled" + }, + { + EXT_TLS + "client_cert_id = \"c\"", + "invalid config: TLS client cert and key secret IDs must be set together" + }, + }; + for (String[] c : cases) { + RuntimeException e = + assertThrows( + c[0], RuntimeException.class, () -> load(PROFILE + c[0] + "\n", new FakeSecrets())); + assertEquals( + c[0], "Client config extension \"aws_secrets_manager\" failed: " + c[1], e.getMessage()); + } + + // Across sources + RuntimeException e = + assertThrows( + RuntimeException.class, + () -> + load( + PROFILE + "api_key = \"key\"\n", + Collections.singletonMap("TEMPORAL_EXT_AWS_SECRETS_MANAGER_API_KEY_ID", "id"), + extension(new FakeSecrets()))); + assertTrue( + e.getMessage(), e.getMessage().endsWith("API key secret ID cannot be set with an API key")); + } + + @Test + public void apiKeySecretIsAllowedWithTlsDisabled() throws Exception { + load( + PROFILE + "[profile.default.tls]\ndisabled = true\n" + EXT + "api_key_id = \"id\"\n", + new FakeSecrets()); + } + + @Test + public void refreshIntervalAndMaxStaleness() throws Exception { + FakeSecrets secrets = new FakeSecrets(); + SecretCache cache = load(API_KEY_PROFILE, secrets).cache; + assertEquals(TimeUnit.MINUTES.toNanos(5), cache.refreshIntervalNanos); + assertEquals(TimeUnit.HOURS.toNanos(24), cache.maxStalenessNanos); + + cache = + plugin( + load( + API_KEY_PROFILE, + Collections.emptyMap(), + extension(secrets) + .setRefreshInterval(Duration.ofMinutes(1)) + .setMaxStaleness(Duration.ZERO))) + .cache; + assertEquals(TimeUnit.MINUTES.toNanos(1), cache.refreshIntervalNanos); + assertEquals(TimeUnit.HOURS.toNanos(24), cache.maxStalenessNanos); + + // Max staleness is at least the refresh interval + cache = + plugin( + load( + API_KEY_PROFILE, + Collections.emptyMap(), + extension(secrets) + .setRefreshInterval(Duration.ofHours(2)) + .setMaxStaleness(Duration.ofHours(1)))) + .cache; + assertEquals(TimeUnit.HOURS.toNanos(2), cache.maxStalenessNanos); + + RuntimeException e = + assertThrows( + RuntimeException.class, + () -> + load( + API_KEY_PROFILE, + Collections.emptyMap(), + extension(secrets).setRefreshInterval(Duration.ofSeconds(-1)))); + assertTrue(e.getMessage(), e.getMessage().endsWith("refreshInterval must not be negative")); + + // From SECRETS_MANAGER_TTL in the process env + assertEquals(0, ttlCache(secrets, "0").refreshIntervalNanos); + assertEquals(TimeUnit.SECONDS.toNanos(600), ttlCache(secrets, "600").refreshIntervalNanos); + // The option takes precedence + assertEquals( + TimeUnit.SECONDS.toNanos(1), + plugin( + load( + API_KEY_PROFILE, + Collections.emptyMap(), + extension(secrets) + .setProcessEnv(Collections.singletonMap("SECRETS_MANAGER_TTL", "600")) + .setRefreshInterval(Duration.ofSeconds(1)))) + .cache + .refreshIntervalNanos); + for (String invalid : new String[] {"-1", "1.5", "abc", "99999999999999999999"}) { + e = assertThrows(RuntimeException.class, () -> ttlCache(secrets, invalid)); + assertEquals( + "Client config extension \"aws_secrets_manager\" failed: invalid SECRETS_MANAGER_TTL \"" + + invalid + + "\", must be a non-negative whole number of seconds", + e.getMessage()); + } + } + + private static SecretCache ttlCache(FakeSecrets secrets, String ttl) throws Exception { + return plugin( + load( + API_KEY_PROFILE, + Collections.emptyMap(), + extension(secrets) + .setProcessEnv(Collections.singletonMap("SECRETS_MANAGER_TTL", ttl)))) + .cache; + } + + @Test + public void configureAppliesSecrets() throws Exception { + FakeSecrets secrets = allSecrets(); + ClientConfigProfile profile = load(ALL_PROFILE, Collections.emptyMap(), extension(secrets)); + WorkflowServiceStubsOptions options = configure(profile); + assertEquals("Bearer secret-key", authorization(options)); + assertNotNull(options.getSslContext()); + assertTrue(options.getSslContext().isClient()); + assertEquals(1, secrets.calls("api-key")); + assertEquals(1, secrets.calls("bundle")); + assertEquals(1, secrets.calls("ca")); + // An API key enables TLS by default, like an API key from envconfig + ClientConfigProfile apiKeyProfile = + load(API_KEY_PROFILE, Collections.emptyMap(), extension(new FakeSecrets().put("id", "k"))); + assertTrue(configure(apiKeyProfile).getEnableHttps()); + } + + @Test + public void configureCombinesSecretsWithProfileTls() throws Exception { + // CA from the profile, client cert and key from a secret + String toml = + PROFILE + + "[profile.default.tls]\nserver_name = \"sni\"\nserver_ca_cert_data = \"\"\"\n" + + TEST_CERT_B + + "\"\"\"\n" + + EXT_TLS + + "client_cert_id = \"bundle\"\nclient_key_id = \"bundle\"\n"; + FakeSecrets secrets = allSecrets(); + WorkflowServiceStubsOptions options = + configure(load(toml, Collections.emptyMap(), extension(secrets))); + assertNotNull(options.getSslContext()); + assertEquals(1, secrets.calls("bundle")); + + // CA from a secret, client cert and key from the profile + toml = + PROFILE + + "[profile.default.tls]\nclient_cert_data = \"\"\"\n" + + TEST_CERT_A + + "\"\"\"\nclient_key_data = \"\"\"\n" + + TEST_KEY_A + + "\"\"\"\n" + + EXT_TLS + + "server_ca_cert_id = \"ca\"\n"; + options = configure(load(toml, Collections.emptyMap(), extension(secrets))); + assertNotNull(options.getSslContext()); + assertEquals(1, secrets.calls("ca")); + } + + @Test + public void configureConflicts() throws Exception { + ClientConfigProfile profile = + load(ALL_PROFILE, Collections.emptyMap(), extension(allSecrets())); + SecretsManagerPlugin plugin = plugin(profile); + + WorkflowServiceStubsOptions.Builder withApiKey = + WorkflowServiceStubsOptions.newBuilder(profile.toWorkflowServiceStubsOptions()) + .addApiKey(() -> "key"); + IllegalStateException e = + assertThrows(IllegalStateException.class, () -> plugin.configureServiceStubs(withApiKey)); + assertEquals( + "cannot use API key from AWS Secrets Manager, API key already set", e.getMessage()); + + // An SSL context that envconfig didn't create + WorkflowServiceStubsOptions.Builder withSsl = + WorkflowServiceStubsOptions.newBuilder(profile.toWorkflowServiceStubsOptions()) + .setSslContext(GrpcSslContexts.forClient().build()); + e = assertThrows(IllegalStateException.class, () -> plugin.configureServiceStubs(withSsl)); + assertEquals( + "cannot use TLS config from AWS Secrets Manager, SSL context already set", e.getMessage()); + } + + @Test + public void configureFailsIfAFetchFails() throws Exception { + ClientConfigProfile profile = + load(API_KEY_PROFILE, Collections.emptyMap(), extension(new FakeSecrets())); + SecretCache.SecretFetchException e = + assertThrows(SecretCache.SecretFetchException.class, () -> configure(profile)); + assertEquals( + "failed fetching AWS Secrets Manager secret \"id\": no secret id", + e.getMessage().replaceAll(" \\(Service:.*", "")); + } + + @Test + public void apiKeyIsRefreshedWhenUsed() throws Exception { + FakeSecrets secrets = new FakeSecrets().put("id", "key1"); + ClientConfigProfile profile = + load( + API_KEY_PROFILE, + Collections.emptyMap(), + extension(secrets).setRefreshInterval(Duration.ZERO)); + plugin(profile).minRefreshIntervalNanos = TimeUnit.MILLISECONDS.toNanos(20); + WorkflowServiceStubsOptions options = configure(profile); + assertEquals("Bearer key1", authorization(options)); + + secrets.put("id", "key2"); + waitUntil(() -> "Bearer key2".equals(authorization(options))); + } + + @Test + public void serviceStubsRunThePlugin() throws Exception { + FakeSecrets secrets = allSecrets(); + ClientConfigProfile profile = load(ALL_PROFILE, Collections.emptyMap(), extension(secrets)); + WorkflowServiceStubs stubs = + WorkflowServiceStubs.newServiceStubs(profile.toWorkflowServiceStubsOptions()); + try { + assertEquals(1, secrets.calls("api-key")); + assertEquals(1, secrets.calls("bundle")); + assertEquals(1, secrets.calls("ca")); + } finally { + stubs.shutdownNow(); + } + } + + @Test + public void apiKeyMetricsUseFinalMetricsScope() throws Exception { + ClientConfigProfile profile = + load(API_KEY_PROFILE, Collections.emptyMap(), extension(new FakeSecrets().put("id", "k"))); + // A metrics plugin registered after this one, so its scope isn't set yet when this plugin + // configures the options + Scope scope = mock(Scope.class, RETURNS_DEEP_STUBS); + WorkflowServiceStubsPlugin metricsPlugin = + new WorkflowServiceStubsPlugin() { + @Nonnull + @Override + public String getName() { + return "test.metrics"; + } + + @Override + public void configureServiceStubs(@Nonnull WorkflowServiceStubsOptions.Builder builder) { + builder.setMetricsScope(scope); + } + + @Nonnull + @Override + public WorkflowServiceStubs connectServiceClient( + @Nonnull WorkflowServiceStubsOptions options, + @Nonnull Supplier next) { + return next.get(); + } + }; + WorkflowServiceStubs stubs = + WorkflowServiceStubs.newServiceStubs( + WorkflowServiceStubsOptions.newBuilder(profile.toWorkflowServiceStubsOptions()) + .setPlugins(plugin(profile), metricsPlugin) + .build()); + try { + verify(scope, atLeastOnce()) + .tagged( + argThat( + tags -> + Telemetry.SECRET_TYPE_API_KEY.equals(tags.get(Telemetry.LABEL_SECRET_TYPE)))); + } finally { + stubs.shutdownNow(); + } + } +} diff --git a/contrib/temporal-aws-secrets-manager/src/test/java/io/temporal/aws/secretsmanager/TestHelpers.java b/contrib/temporal-aws-secrets-manager/src/test/java/io/temporal/aws/secretsmanager/TestHelpers.java new file mode 100644 index 0000000000..ec9ecc73f8 --- /dev/null +++ b/contrib/temporal-aws-secrets-manager/src/test/java/io/temporal/aws/secretsmanager/TestHelpers.java @@ -0,0 +1,183 @@ +package io.temporal.aws.secretsmanager; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BooleanSupplier; + +final class TestHelpers { + private TestHelpers() {} + + // Self-signed EC test certs, generated just for this module's tests and deliberately expired so + // they can't be used as real credentials. They're only parsed and matched against their keys, + // never used for TLS. TEST_KEY_A_EC is TEST_KEY_A in the traditional EC format. + + static final String TEST_CERT_A = + "-----BEGIN CERTIFICATE-----\n" + + "MIIBnjCCAUWgAwIBAgIUDqZuJ4EivyOw4rVmjlePA7udGhUwCgYIKoZIzj0EAwIw\n" + + "JTEjMCEGA1UEAwwaYXdzLXNlY3JldHMtbWFuYWdlci10ZXN0LWEwHhcNMDAwMTAx\n" + + "MDAwMDAwWhcNMDAwMTAyMDAwMDAwWjAlMSMwIQYDVQQDDBphd3Mtc2VjcmV0cy1t\n" + + "YW5hZ2VyLXRlc3QtYTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABPYjw38P+CZ7\n" + + "5Gn5L4yZ7QfLq1MNax7wuqmksAu+SSwNwmjiNBM0LpvEHZGaB5uun4JVwPDX2Caw\n" + + "dOzRMMjkJfGjUzBRMB0GA1UdDgQWBBQqh6LjSKZlTFBKw8fPrKcMppVc0TAfBgNV\n" + + "HSMEGDAWgBQqh6LjSKZlTFBKw8fPrKcMppVc0TAPBgNVHRMBAf8EBTADAQH/MAoG\n" + + "CCqGSM49BAMCA0cAMEQCIBOvMndUhN/E4pi3ohOrZhZ98yUG+5wGkPirzSgiyrGA\n" + + "AiB67ITqPqGBwT1mOTKnHa0R801etROwSp8EicjU65Xi0A==\n" + + "-----END CERTIFICATE-----\n"; + + static final String TEST_KEY_A = + "-----BEGIN PRIVATE KEY-----\n" + + "MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQguBxMHVTJpKDUQdXp\n" + + "lUqPRKgtTMRVEVjYJbRybzKMYoehRANCAAT2I8N/D/gme+Rp+S+Mme0Hy6tTDWse\n" + + "8LqppLALvkksDcJo4jQTNC6bxB2Rmgebrp+CVcDw19gmsHTs0TDI5CXx\n" + + "-----END PRIVATE KEY-----\n"; + + static final String TEST_KEY_A_EC = + "-----BEGIN EC PRIVATE KEY-----\n" + + "MHcCAQEEILgcTB1UyaSg1EHV6ZVKj0SoLUzEVRFY2CW0cm8yjGKHoAoGCCqGSM49\n" + + "AwEHoUQDQgAE9iPDfw/4JnvkafkvjJntB8urUw1rHvC6qaSwC75JLA3CaOI0EzQu\n" + + "m8QdkZoHm66fglXA8NfYJrB07NEwyOQl8Q==\n" + + "-----END EC PRIVATE KEY-----\n"; + + static final String TEST_CERT_B = + "-----BEGIN CERTIFICATE-----\n" + + "MIIBoDCCAUWgAwIBAgIUJkXwBu3Peti5+WEGvCfk3YQYnFgwCgYIKoZIzj0EAwIw\n" + + "JTEjMCEGA1UEAwwaYXdzLXNlY3JldHMtbWFuYWdlci10ZXN0LWIwHhcNMDAwMTAx\n" + + "MDAwMDAwWhcNMDAwMTAyMDAwMDAwWjAlMSMwIQYDVQQDDBphd3Mtc2VjcmV0cy1t\n" + + "YW5hZ2VyLXRlc3QtYjBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABFeVEzhEhUbs\n" + + "l4sgc8vkm5Lfgemqc5h2EirK5OAna/9/Yd4feUw8JYK6VKfIeJJBVeaKjJbihaTY\n" + + "aXA9zg+PBMCjUzBRMB0GA1UdDgQWBBTBGCIL3D2zvsou4vFsR2lsmwTtdjAfBgNV\n" + + "HSMEGDAWgBTBGCIL3D2zvsou4vFsR2lsmwTtdjAPBgNVHRMBAf8EBTADAQH/MAoG\n" + + "CCqGSM49BAMCA0kAMEYCIQDnP+D+kffMeiRoerQaNplgBlaH++eFY+MPLKMLMuRc\n" + + "UAIhAKcdRK2djsmk2QP7q7yc1js9HeDONluoo2WaWLMilAyk\n" + + "-----END CERTIFICATE-----\n"; + + static final String TEST_KEY_B = + "-----BEGIN PRIVATE KEY-----\n" + + "MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgsjA2Q9nct3Mb+KOR\n" + + "akioJrOmqRw6/SzLSucTqopek9qhRANCAARXlRM4RIVG7JeLIHPL5JuS34HpqnOY\n" + + "dhIqyuTgJ2v/f2HeH3lMPCWCulSnyHiSQVXmioyW4oWk2GlwPc4PjwTA\n" + + "-----END PRIVATE KEY-----\n"; + + /** Polls until {@code cond} is true, failing if it isn't within 5 seconds. */ + static void waitUntil(BooleanSupplier cond) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (!cond.getAsBoolean()) { + if (System.nanoTime() > deadline) { + throw new AssertionError("condition not met within 5s"); + } + Thread.sleep(10); + } + } + + /** + * Returns configured values, counting fetches. A value may be an exception to throw. If {@code + * gate} is set, fetches wait for it. + */ + static final class FakeFetcher implements Fetcher { + final Map values = new ConcurrentHashMap<>(); + final Map calls = new ConcurrentHashMap<>(); + volatile CountDownLatch gate; + private final String path; + + FakeFetcher() { + this(Telemetry.FETCH_PATH_SDK); + } + + FakeFetcher(String path) { + this.path = path; + } + + void set(String secretId, String value, String versionId) { + set(secretId, value.getBytes(java.nio.charset.StandardCharsets.UTF_8), versionId); + } + + void set(String secretId, byte[] value, String versionId) { + values.put(secretId, new SecretValue(value, versionId)); + } + + void fail(String secretId, Exception err) { + values.put(secretId, err); + } + + int calls(String secretId) { + AtomicInteger n = calls.get(secretId); + return n != null ? n.get() : 0; + } + + @Override + public String path() { + return path; + } + + @Override + public SecretValue fetch(String secretId) throws Exception { + calls.computeIfAbsent(secretId, k -> new AtomicInteger()).incrementAndGet(); + CountDownLatch g = gate; + if (g != null && !g.await(5, TimeUnit.SECONDS)) { + throw new AssertionError("gate not opened"); + } + Object value = values.get(secretId); + if (value == null) { + throw new Exception("no fake value for " + secretId); + } + if (value instanceof Exception) { + throw (Exception) value; + } + return (SecretValue) value; + } + } + + /** A recorded metric update. */ + static final class MetricUpdate { + final String name; + final Map tags; + + MetricUpdate(String name, Map tags) { + this.name = name; + this.tags = tags; + } + } + + /** Telemetry that records metric updates. */ + static final class RecordingTelemetry extends Telemetry { + final List updates = new ArrayList<>(); + + RecordingTelemetry() { + super(null); + } + + @Override + synchronized void cacheLookup(Map tags) { + updates.add(new MetricUpdate(METRIC_CACHE_LOOKUP, tags)); + } + + @Override + synchronized void fetchFailed(Map tags) { + updates.add(new MetricUpdate(METRIC_FETCH_FAILED, tags)); + } + + @Override + synchronized void fetchLatency(Map tags, long nanos) { + updates.add(new MetricUpdate(METRIC_FETCH_LATENCY, tags)); + } + + /** Returns and clears the tags of each update of the given metric. */ + synchronized List> take(String name) { + List> taken = new ArrayList<>(); + updates.removeIf( + u -> { + if (u.name.equals(name)) { + taken.add(u.tags); + return true; + } + return false; + }); + return taken; + } + } +} diff --git a/contrib/temporal-payload-storage-s3driver-awssdkv2/build.gradle b/contrib/temporal-payload-storage-s3driver-awssdkv2/build.gradle index a6e57058cc..f0d5eca05e 100644 --- a/contrib/temporal-payload-storage-s3driver-awssdkv2/build.gradle +++ b/contrib/temporal-payload-storage-s3driver-awssdkv2/build.gradle @@ -1,9 +1,5 @@ description = '''Temporal Java SDK External Storage S3 Driver - AWS SDK v2 Client''' -ext { - awsSdkVersion = '2.31.0' -} - dependencies { api project(':temporal-payload-storage-s3driver') api platform("software.amazon.awssdk:bom:$awsSdkVersion") diff --git a/settings.gradle b/settings.gradle index a2ee73e91a..77c0d18a74 100644 --- a/settings.gradle +++ b/settings.gradle @@ -28,6 +28,8 @@ include 'temporal-workflowstreams' project(':temporal-workflowstreams').projectDir = file('contrib/temporal-workflowstreams') include 'temporal-aws-lambda' project(':temporal-aws-lambda').projectDir = file('contrib/temporal-aws-lambda') +include 'temporal-aws-secrets-manager' +project(':temporal-aws-secrets-manager').projectDir = file('contrib/temporal-aws-secrets-manager') include 'temporal-gcp-cloud-run-opentelemetry' project(':temporal-gcp-cloud-run-opentelemetry').projectDir = file('contrib/temporal-gcp-cloud-run-opentelemetry') include 'temporal-gcp-cloud-run-id' diff --git a/temporal-bom/build.gradle b/temporal-bom/build.gradle index 0bbae3e4bd..ef175819b8 100644 --- a/temporal-bom/build.gradle +++ b/temporal-bom/build.gradle @@ -10,6 +10,7 @@ dependencies { api project(':temporal-opentelemetry') api project(':temporal-opentracing') api project(':temporal-aws-lambda') + api project(':temporal-aws-secrets-manager') api project(':temporal-gcp-cloud-run-opentelemetry') api project(':temporal-gcp-cloud-run-id') api project(':temporal-remote-data-encoder')