diff --git a/README.md b/README.md index 0ceddc8f..7d3a1034 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,13 @@ mvn clean package More thorough usage examples are still TODO, but there is a basic test implementation in [TestMain](server/src/test/java/io/envoyproxy/controlplane/server/TestMain.java). +[CanaryRolloutExample](server/src/test/java/io/envoyproxy/controlplane/server/example/CanaryRolloutExample.java) +shows a staged rollout: one snapshot per group of Envoys (keyed by `node.cluster`), a new version +pushed to a canary group first, and promoted only after every canary Envoy has applied it. +[AckTracker](server/src/test/java/io/envoyproxy/controlplane/server/example/AckTracker.java), a +`DiscoveryServerCallbacks`, learns from the ACKs and NACKs in the xDS stream which version each +Envoy has applied, so the control plane can wait for "applied" rather than "pushed". + ### Envoy API There is automation built into this repo to keep the Envoy API (i.e. protobuf files) up to date with the latest available Envoy release. diff --git a/server/src/test/java/io/envoyproxy/controlplane/server/example/AckTracker.java b/server/src/test/java/io/envoyproxy/controlplane/server/example/AckTracker.java new file mode 100644 index 00000000..b6d86470 --- /dev/null +++ b/server/src/test/java/io/envoyproxy/controlplane/server/example/AckTracker.java @@ -0,0 +1,159 @@ +package io.envoyproxy.controlplane.server.example; + +import io.envoyproxy.controlplane.server.DiscoveryServerCallbacks; +import io.envoyproxy.envoy.service.discovery.v3.DeltaDiscoveryRequest; +import io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest; +import io.envoyproxy.envoy.service.discovery.v3.DiscoveryResponse; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.ReentrantLock; + +/** + * Learns which version of each resource type every connected Envoy has actually applied, from + * the xDS conversation itself. + * + *

In state-of-the-world xDS, Envoy answers every {@link DiscoveryResponse} with a new {@link + * DiscoveryRequest}. If {@code error_detail} is empty it is an ACK, and {@code version_info} is + * the version Envoy applied. If {@code error_detail} is set it is a NACK: Envoy rejected the + * response whose nonce it echoes, and keeps serving its previous config. + * + *

{@code SimpleCache.setSnapshot} only means "available"; waiting on this tracker means + * "applied". A control plane that rolls config out in stages (for example canary first) should + * wait for the ACK before judging or promoting the change. Register it with {@code new + * V3DiscoveryServer(tracker, cache)}. + */ +public class AckTracker implements DiscoveryServerCallbacks { + + /** The outcome of waiting for a version. */ + public enum Status { + /** Every connected Envoy in the group applied the version. */ + ACKED, + /** At least one Envoy in the group rejected the version. */ + NACKED, + /** Some Envoy has not answered yet. */ + TIMEOUT, + /** No Envoy in the group is connected. */ + NO_NODES + } + + /** What one ADS stream (one Envoy) has told us. */ + private static final class Stream { + volatile String group; + /** type URL -> last version ACKed / NACKed. */ + final Map acked = new ConcurrentHashMap<>(); + final Map nacked = new ConcurrentHashMap<>(); + /** response nonce -> version sent with it, so a NACK can name the version it rejects. */ + final Map sent = new ConcurrentHashMap<>(); + } + + private final Map streams = new ConcurrentHashMap<>(); + private final ReentrantLock lock = new ReentrantLock(); + private final Condition changed = lock.newCondition(); + + @Override + public void onStreamOpen(long streamId, String typeUrl) { + streams.put(streamId, new Stream()); + } + + @Override + public void onStreamClose(long streamId, String typeUrl) { + streams.remove(streamId); + signal(); + } + + @Override + public void onStreamCloseWithError(long streamId, String typeUrl, Throwable error) { + streams.remove(streamId); + signal(); + } + + @Override + public void onV3StreamResponse( + long streamId, DiscoveryRequest request, DiscoveryResponse response) { + Stream s = streams.get(streamId); + if (s != null) { + s.sent.put(response.getNonce(), response.getVersionInfo()); + } + } + + @Override + public void onV3StreamRequest(long streamId, DiscoveryRequest request) { + Stream s = streams.computeIfAbsent(streamId, id -> new Stream()); + if (request.hasNode()) { + s.group = request.getNode().getCluster(); + } + if (request.getResponseNonce().isEmpty()) { + return; // the first request for a type is a subscription, not a reply + } + String sentVersion = s.sent.remove(request.getResponseNonce()); + if (request.hasErrorDetail()) { + if (sentVersion != null) { + s.nacked.put(request.getTypeUrl(), sentVersion); + } + } else { + s.acked.put(request.getTypeUrl(), request.getVersionInfo()); + } + signal(); + } + + @Override + public void onV3StreamDeltaRequest(long streamId, DeltaDiscoveryRequest request) { + // This example tracks state-of-the-world xDS only. + } + + /** + * Waits until every Envoy connected in {@code group} has ACKed {@code version} of {@code + * typeUrl}, any of them NACKs it, or the timeout passes. Pass the latest version you pushed: + * versions are compared for equality, since snapshot versions are opaque strings. + */ + public Status await(String group, String typeUrl, String version, long timeout, TimeUnit unit) + throws InterruptedException { + long deadline = System.nanoTime() + unit.toNanos(timeout); + lock.lock(); + try { + while (true) { + Status now = status(group, typeUrl, version); + if (now == Status.ACKED || now == Status.NACKED) { + return now; + } + long left = deadline - System.nanoTime(); + if (left <= 0) { + return now; + } + changed.awaitNanos(left); + } + } finally { + lock.unlock(); + } + } + + private Status status(String group, String typeUrl, String version) { + boolean any = false; + boolean all = true; + for (Stream s : streams.values()) { + if (!group.equals(s.group)) { + continue; + } + any = true; + if (version.equals(s.nacked.get(typeUrl))) { + return Status.NACKED; + } + all &= version.equals(s.acked.get(typeUrl)); + } + if (!any) { + return Status.NO_NODES; + } + return all ? Status.ACKED : Status.TIMEOUT; + } + + private void signal() { + lock.lock(); + try { + changed.signalAll(); + } finally { + lock.unlock(); + } + } +} diff --git a/server/src/test/java/io/envoyproxy/controlplane/server/example/AckTrackerTest.java b/server/src/test/java/io/envoyproxy/controlplane/server/example/AckTrackerTest.java new file mode 100644 index 00000000..d4117b51 --- /dev/null +++ b/server/src/test/java/io/envoyproxy/controlplane/server/example/AckTrackerTest.java @@ -0,0 +1,94 @@ +package io.envoyproxy.controlplane.server.example; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.google.rpc.Status; +import io.envoyproxy.controlplane.cache.Resources; +import io.envoyproxy.envoy.config.core.v3.Node; +import io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest; +import io.envoyproxy.envoy.service.discovery.v3.DiscoveryResponse; +import java.util.concurrent.TimeUnit; +import org.junit.Test; + +public class AckTrackerTest { + + private static final String CLUSTERS = Resources.V3.CLUSTER_TYPE_URL; + + private final AckTracker tracker = new AckTracker(); + + private void connect(long stream, String group) { + tracker.onStreamOpen(stream, ""); + tracker.onV3StreamRequest( + stream, + DiscoveryRequest.newBuilder() + .setNode(Node.newBuilder().setId("envoy-" + stream).setCluster(group)) + .setTypeUrl(CLUSTERS) + .build()); + } + + private void send(long stream, String version) { + tracker.onV3StreamResponse( + stream, + DiscoveryRequest.getDefaultInstance(), + DiscoveryResponse.newBuilder() + .setTypeUrl(CLUSTERS) + .setVersionInfo(version) + .setNonce("n" + version) + .build()); + } + + private void ack(long stream, String version) { + tracker.onV3StreamRequest( + stream, + DiscoveryRequest.newBuilder() + .setTypeUrl(CLUSTERS) + .setVersionInfo(version) + .setResponseNonce("n" + version) + .build()); + } + + private void nack(long stream, String version, String stillOn) { + tracker.onV3StreamRequest( + stream, + DiscoveryRequest.newBuilder() + .setTypeUrl(CLUSTERS) + .setVersionInfo(stillOn) + .setResponseNonce("n" + version) + .setErrorDetail(Status.newBuilder().setMessage("invalid cluster")) + .build()); + } + + private AckTracker.Status await(String group, String version) throws InterruptedException { + return tracker.await(group, CLUSTERS, version, 100, TimeUnit.MILLISECONDS); + } + + @Test + public void ackedOnlyWhenEveryEnvoyInTheGroupAppliedTheVersion() throws Exception { + connect(1, "canary"); + connect(2, "canary"); + send(1, "2"); + send(2, "2"); + ack(1, "2"); + assertThat(await("canary", "2")).isEqualTo(AckTracker.Status.TIMEOUT); + ack(2, "2"); + assertThat(await("canary", "2")).isEqualTo(AckTracker.Status.ACKED); + } + + @Test + public void nackedWhenAnyEnvoyRejectsTheVersion() throws Exception { + connect(1, "canary"); + send(1, "2"); + nack(1, "2", "1"); + assertThat(await("canary", "2")).isEqualTo(AckTracker.Status.NACKED); + } + + @Test + public void otherGroupsAndClosedStreamsDoNotCount() throws Exception { + connect(1, "stable"); + send(1, "2"); + ack(1, "2"); + assertThat(await("canary", "2")).isEqualTo(AckTracker.Status.NO_NODES); + tracker.onStreamClose(1, ""); + assertThat(await("stable", "2")).isEqualTo(AckTracker.Status.NO_NODES); + } +} diff --git a/server/src/test/java/io/envoyproxy/controlplane/server/example/CanaryRolloutExample.java b/server/src/test/java/io/envoyproxy/controlplane/server/example/CanaryRolloutExample.java new file mode 100644 index 00000000..81e6ee69 --- /dev/null +++ b/server/src/test/java/io/envoyproxy/controlplane/server/example/CanaryRolloutExample.java @@ -0,0 +1,83 @@ +package io.envoyproxy.controlplane.server.example; + +import com.google.common.collect.ImmutableList; +import io.envoyproxy.controlplane.cache.Resources; +import io.envoyproxy.controlplane.cache.TestResources; +import io.envoyproxy.controlplane.cache.v3.SimpleCache; +import io.envoyproxy.controlplane.cache.v3.Snapshot; +import io.envoyproxy.controlplane.server.V3DiscoveryServer; +import io.envoyproxy.envoy.config.cluster.v3.Cluster; +import io.grpc.Server; +import io.grpc.netty.NettyServerBuilder; +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +/** + * Example: roll a config change out to a canary group of Envoys first, wait until every canary + * Envoy has applied it (ACK) or rejected it (NACK), and only then promote it to the rest. + * + *

Envoys are grouped by {@code node.cluster}, which Envoy sets from {@code --service-cluster}. + * Start one Envoy with {@code --service-cluster canary} and one with {@code --service-cluster + * stable}, both with an ADS bootstrap pointing at port 12345. + * + *

A real control plane would also check the canary's health (synthetic requests, or its error + * rate from metrics) after the ACK and before promoting. A NACK, a timeout or no connected canary + * all mean: do not promote, and put the canary back on the last good snapshot. + */ +public class CanaryRolloutExample { + + private static final String CANARY = "canary"; + private static final String STABLE = "stable"; + + /** Runs the example; the server stays up until the process is stopped. */ + public static void main(String[] args) throws IOException, InterruptedException { + // One snapshot per group, chosen by the Envoy's node.cluster. + SimpleCache cache = new SimpleCache<>(node -> node.getCluster()); + AckTracker acks = new AckTracker(); + V3DiscoveryServer discoveryServer = new V3DiscoveryServer(acks, cache); + + Server server = + NettyServerBuilder.forPort(12345) + .addService(discoveryServer.getAggregatedDiscoveryServiceImpl()) + .build() + .start(); + Runtime.getRuntime().addShutdownHook(new Thread(server::shutdown)); + System.out.println("xDS server listening on port " + server.getPort()); + + Snapshot good = snapshot("cluster0", 1234, "1"); + cache.setSnapshot(CANARY, good); + cache.setSnapshot(STABLE, good); + + // Give the Envoys time to connect, then roll out version 2. + Thread.sleep(10_000); + Snapshot candidate = snapshot("cluster1", 1235, "2"); + cache.setSnapshot(CANARY, candidate); + + AckTracker.Status status = + acks.await(CANARY, Resources.V3.CLUSTER_TYPE_URL, "2", 10, TimeUnit.SECONDS); + System.out.println("Canary answered version 2 with " + status); + + if (status == AckTracker.Status.ACKED) { + // Here a real control plane would probe the canary before promoting. + cache.setSnapshot(STABLE, candidate); + System.out.println("Promoted version 2 to " + STABLE); + } else { + // A new version string, so the canary is sent the last good config again. + cache.setSnapshot(CANARY, snapshot("cluster0", 1234, "3")); + System.out.println("Rolled the canary back; " + STABLE + " never saw version 2"); + } + + server.awaitTermination(); + } + + private static Snapshot snapshot(String cluster, int port, String version) { + return Snapshot.create( + ImmutableList.of( + TestResources.createCluster(cluster, "127.0.0.1", port, Cluster.DiscoveryType.STATIC)), + ImmutableList.of(), + ImmutableList.of(), + ImmutableList.of(), + ImmutableList.of(), + version); + } +}