From 899173c3b1111280ac81fc855a92a3988d6ef309 Mon Sep 17 00:00:00 2001
From: Vikram Subramanya <5620531+vikrams3@users.noreply.github.com>
Date: Wed, 11 Nov 2026 13:42:05 -0800
Subject: [PATCH] examples: ACK-aware canary rollout
Add a usage example for staged rollouts, since the README notes that
usage examples are still TODO:
- AckTracker: a DiscoveryServerCallbacks that records, per stream and
resource type, the version each Envoy ACKed or NACKed, and lets the
control plane wait until every Envoy in a node group applied a version
- CanaryRolloutExample: one snapshot per node.cluster group; push to a
canary group, wait for its ACK, then promote or roll back
- AckTrackerTest: ACK from every node, NACK, other groups, closed streams
Java 8 compatible; test sources only, no change to the library.
Signed-off-by: Vikram Subramanya <5620531+vikrams3@users.noreply.github.com>
---
README.md | 7 +
.../server/example/AckTracker.java | 159 ++++++++++++++++++
.../server/example/AckTrackerTest.java | 94 +++++++++++
.../server/example/CanaryRolloutExample.java | 83 +++++++++
4 files changed, 343 insertions(+)
create mode 100644 server/src/test/java/io/envoyproxy/controlplane/server/example/AckTracker.java
create mode 100644 server/src/test/java/io/envoyproxy/controlplane/server/example/AckTrackerTest.java
create mode 100644 server/src/test/java/io/envoyproxy/controlplane/server/example/CanaryRolloutExample.java
diff --git a/README.md b/README.md
index 0ceddc8f2..7d3a1034f 100644
--- a/README.md
+++ b/README.md
@@ -28,6 +28,13 @@ mvn clean package
More thorough usage examples are still TODO, but there is a basic test implementation in
[TestMain](server/src/test/java/io/envoyproxy/controlplane/server/TestMain.java).
+[CanaryRolloutExample](server/src/test/java/io/envoyproxy/controlplane/server/example/CanaryRolloutExample.java)
+shows a staged rollout: one snapshot per group of Envoys (keyed by `node.cluster`), a new version
+pushed to a canary group first, and promoted only after every canary Envoy has applied it.
+[AckTracker](server/src/test/java/io/envoyproxy/controlplane/server/example/AckTracker.java), a
+`DiscoveryServerCallbacks`, learns from the ACKs and NACKs in the xDS stream which version each
+Envoy has applied, so the control plane can wait for "applied" rather than "pushed".
+
### Envoy API
There is automation built into this repo to keep the Envoy API (i.e. protobuf files)
up to date with the latest available Envoy release.
diff --git a/server/src/test/java/io/envoyproxy/controlplane/server/example/AckTracker.java b/server/src/test/java/io/envoyproxy/controlplane/server/example/AckTracker.java
new file mode 100644
index 000000000..b6d864704
--- /dev/null
+++ b/server/src/test/java/io/envoyproxy/controlplane/server/example/AckTracker.java
@@ -0,0 +1,159 @@
+package io.envoyproxy.controlplane.server.example;
+
+import io.envoyproxy.controlplane.server.DiscoveryServerCallbacks;
+import io.envoyproxy.envoy.service.discovery.v3.DeltaDiscoveryRequest;
+import io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest;
+import io.envoyproxy.envoy.service.discovery.v3.DiscoveryResponse;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.ReentrantLock;
+
+/**
+ * Learns which version of each resource type every connected Envoy has actually applied, from
+ * the xDS conversation itself.
+ *
+ *
In state-of-the-world xDS, Envoy answers every {@link DiscoveryResponse} with a new {@link
+ * DiscoveryRequest}. If {@code error_detail} is empty it is an ACK, and {@code version_info} is
+ * the version Envoy applied. If {@code error_detail} is set it is a NACK: Envoy rejected the
+ * response whose nonce it echoes, and keeps serving its previous config.
+ *
+ *
{@code SimpleCache.setSnapshot} only means "available"; waiting on this tracker means
+ * "applied". A control plane that rolls config out in stages (for example canary first) should
+ * wait for the ACK before judging or promoting the change. Register it with {@code new
+ * V3DiscoveryServer(tracker, cache)}.
+ */
+public class AckTracker implements DiscoveryServerCallbacks {
+
+ /** The outcome of waiting for a version. */
+ public enum Status {
+ /** Every connected Envoy in the group applied the version. */
+ ACKED,
+ /** At least one Envoy in the group rejected the version. */
+ NACKED,
+ /** Some Envoy has not answered yet. */
+ TIMEOUT,
+ /** No Envoy in the group is connected. */
+ NO_NODES
+ }
+
+ /** What one ADS stream (one Envoy) has told us. */
+ private static final class Stream {
+ volatile String group;
+ /** type URL -> last version ACKed / NACKed. */
+ final Map acked = new ConcurrentHashMap<>();
+ final Map nacked = new ConcurrentHashMap<>();
+ /** response nonce -> version sent with it, so a NACK can name the version it rejects. */
+ final Map sent = new ConcurrentHashMap<>();
+ }
+
+ private final Map streams = new ConcurrentHashMap<>();
+ private final ReentrantLock lock = new ReentrantLock();
+ private final Condition changed = lock.newCondition();
+
+ @Override
+ public void onStreamOpen(long streamId, String typeUrl) {
+ streams.put(streamId, new Stream());
+ }
+
+ @Override
+ public void onStreamClose(long streamId, String typeUrl) {
+ streams.remove(streamId);
+ signal();
+ }
+
+ @Override
+ public void onStreamCloseWithError(long streamId, String typeUrl, Throwable error) {
+ streams.remove(streamId);
+ signal();
+ }
+
+ @Override
+ public void onV3StreamResponse(
+ long streamId, DiscoveryRequest request, DiscoveryResponse response) {
+ Stream s = streams.get(streamId);
+ if (s != null) {
+ s.sent.put(response.getNonce(), response.getVersionInfo());
+ }
+ }
+
+ @Override
+ public void onV3StreamRequest(long streamId, DiscoveryRequest request) {
+ Stream s = streams.computeIfAbsent(streamId, id -> new Stream());
+ if (request.hasNode()) {
+ s.group = request.getNode().getCluster();
+ }
+ if (request.getResponseNonce().isEmpty()) {
+ return; // the first request for a type is a subscription, not a reply
+ }
+ String sentVersion = s.sent.remove(request.getResponseNonce());
+ if (request.hasErrorDetail()) {
+ if (sentVersion != null) {
+ s.nacked.put(request.getTypeUrl(), sentVersion);
+ }
+ } else {
+ s.acked.put(request.getTypeUrl(), request.getVersionInfo());
+ }
+ signal();
+ }
+
+ @Override
+ public void onV3StreamDeltaRequest(long streamId, DeltaDiscoveryRequest request) {
+ // This example tracks state-of-the-world xDS only.
+ }
+
+ /**
+ * Waits until every Envoy connected in {@code group} has ACKed {@code version} of {@code
+ * typeUrl}, any of them NACKs it, or the timeout passes. Pass the latest version you pushed:
+ * versions are compared for equality, since snapshot versions are opaque strings.
+ */
+ public Status await(String group, String typeUrl, String version, long timeout, TimeUnit unit)
+ throws InterruptedException {
+ long deadline = System.nanoTime() + unit.toNanos(timeout);
+ lock.lock();
+ try {
+ while (true) {
+ Status now = status(group, typeUrl, version);
+ if (now == Status.ACKED || now == Status.NACKED) {
+ return now;
+ }
+ long left = deadline - System.nanoTime();
+ if (left <= 0) {
+ return now;
+ }
+ changed.awaitNanos(left);
+ }
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ private Status status(String group, String typeUrl, String version) {
+ boolean any = false;
+ boolean all = true;
+ for (Stream s : streams.values()) {
+ if (!group.equals(s.group)) {
+ continue;
+ }
+ any = true;
+ if (version.equals(s.nacked.get(typeUrl))) {
+ return Status.NACKED;
+ }
+ all &= version.equals(s.acked.get(typeUrl));
+ }
+ if (!any) {
+ return Status.NO_NODES;
+ }
+ return all ? Status.ACKED : Status.TIMEOUT;
+ }
+
+ private void signal() {
+ lock.lock();
+ try {
+ changed.signalAll();
+ } finally {
+ lock.unlock();
+ }
+ }
+}
diff --git a/server/src/test/java/io/envoyproxy/controlplane/server/example/AckTrackerTest.java b/server/src/test/java/io/envoyproxy/controlplane/server/example/AckTrackerTest.java
new file mode 100644
index 000000000..d4117b516
--- /dev/null
+++ b/server/src/test/java/io/envoyproxy/controlplane/server/example/AckTrackerTest.java
@@ -0,0 +1,94 @@
+package io.envoyproxy.controlplane.server.example;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import com.google.rpc.Status;
+import io.envoyproxy.controlplane.cache.Resources;
+import io.envoyproxy.envoy.config.core.v3.Node;
+import io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest;
+import io.envoyproxy.envoy.service.discovery.v3.DiscoveryResponse;
+import java.util.concurrent.TimeUnit;
+import org.junit.Test;
+
+public class AckTrackerTest {
+
+ private static final String CLUSTERS = Resources.V3.CLUSTER_TYPE_URL;
+
+ private final AckTracker tracker = new AckTracker();
+
+ private void connect(long stream, String group) {
+ tracker.onStreamOpen(stream, "");
+ tracker.onV3StreamRequest(
+ stream,
+ DiscoveryRequest.newBuilder()
+ .setNode(Node.newBuilder().setId("envoy-" + stream).setCluster(group))
+ .setTypeUrl(CLUSTERS)
+ .build());
+ }
+
+ private void send(long stream, String version) {
+ tracker.onV3StreamResponse(
+ stream,
+ DiscoveryRequest.getDefaultInstance(),
+ DiscoveryResponse.newBuilder()
+ .setTypeUrl(CLUSTERS)
+ .setVersionInfo(version)
+ .setNonce("n" + version)
+ .build());
+ }
+
+ private void ack(long stream, String version) {
+ tracker.onV3StreamRequest(
+ stream,
+ DiscoveryRequest.newBuilder()
+ .setTypeUrl(CLUSTERS)
+ .setVersionInfo(version)
+ .setResponseNonce("n" + version)
+ .build());
+ }
+
+ private void nack(long stream, String version, String stillOn) {
+ tracker.onV3StreamRequest(
+ stream,
+ DiscoveryRequest.newBuilder()
+ .setTypeUrl(CLUSTERS)
+ .setVersionInfo(stillOn)
+ .setResponseNonce("n" + version)
+ .setErrorDetail(Status.newBuilder().setMessage("invalid cluster"))
+ .build());
+ }
+
+ private AckTracker.Status await(String group, String version) throws InterruptedException {
+ return tracker.await(group, CLUSTERS, version, 100, TimeUnit.MILLISECONDS);
+ }
+
+ @Test
+ public void ackedOnlyWhenEveryEnvoyInTheGroupAppliedTheVersion() throws Exception {
+ connect(1, "canary");
+ connect(2, "canary");
+ send(1, "2");
+ send(2, "2");
+ ack(1, "2");
+ assertThat(await("canary", "2")).isEqualTo(AckTracker.Status.TIMEOUT);
+ ack(2, "2");
+ assertThat(await("canary", "2")).isEqualTo(AckTracker.Status.ACKED);
+ }
+
+ @Test
+ public void nackedWhenAnyEnvoyRejectsTheVersion() throws Exception {
+ connect(1, "canary");
+ send(1, "2");
+ nack(1, "2", "1");
+ assertThat(await("canary", "2")).isEqualTo(AckTracker.Status.NACKED);
+ }
+
+ @Test
+ public void otherGroupsAndClosedStreamsDoNotCount() throws Exception {
+ connect(1, "stable");
+ send(1, "2");
+ ack(1, "2");
+ assertThat(await("canary", "2")).isEqualTo(AckTracker.Status.NO_NODES);
+ tracker.onStreamClose(1, "");
+ assertThat(await("stable", "2")).isEqualTo(AckTracker.Status.NO_NODES);
+ }
+}
diff --git a/server/src/test/java/io/envoyproxy/controlplane/server/example/CanaryRolloutExample.java b/server/src/test/java/io/envoyproxy/controlplane/server/example/CanaryRolloutExample.java
new file mode 100644
index 000000000..81e6ee69f
--- /dev/null
+++ b/server/src/test/java/io/envoyproxy/controlplane/server/example/CanaryRolloutExample.java
@@ -0,0 +1,83 @@
+package io.envoyproxy.controlplane.server.example;
+
+import com.google.common.collect.ImmutableList;
+import io.envoyproxy.controlplane.cache.Resources;
+import io.envoyproxy.controlplane.cache.TestResources;
+import io.envoyproxy.controlplane.cache.v3.SimpleCache;
+import io.envoyproxy.controlplane.cache.v3.Snapshot;
+import io.envoyproxy.controlplane.server.V3DiscoveryServer;
+import io.envoyproxy.envoy.config.cluster.v3.Cluster;
+import io.grpc.Server;
+import io.grpc.netty.NettyServerBuilder;
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Example: roll a config change out to a canary group of Envoys first, wait until every canary
+ * Envoy has applied it (ACK) or rejected it (NACK), and only then promote it to the rest.
+ *
+ * Envoys are grouped by {@code node.cluster}, which Envoy sets from {@code --service-cluster}.
+ * Start one Envoy with {@code --service-cluster canary} and one with {@code --service-cluster
+ * stable}, both with an ADS bootstrap pointing at port 12345.
+ *
+ *
A real control plane would also check the canary's health (synthetic requests, or its error
+ * rate from metrics) after the ACK and before promoting. A NACK, a timeout or no connected canary
+ * all mean: do not promote, and put the canary back on the last good snapshot.
+ */
+public class CanaryRolloutExample {
+
+ private static final String CANARY = "canary";
+ private static final String STABLE = "stable";
+
+ /** Runs the example; the server stays up until the process is stopped. */
+ public static void main(String[] args) throws IOException, InterruptedException {
+ // One snapshot per group, chosen by the Envoy's node.cluster.
+ SimpleCache cache = new SimpleCache<>(node -> node.getCluster());
+ AckTracker acks = new AckTracker();
+ V3DiscoveryServer discoveryServer = new V3DiscoveryServer(acks, cache);
+
+ Server server =
+ NettyServerBuilder.forPort(12345)
+ .addService(discoveryServer.getAggregatedDiscoveryServiceImpl())
+ .build()
+ .start();
+ Runtime.getRuntime().addShutdownHook(new Thread(server::shutdown));
+ System.out.println("xDS server listening on port " + server.getPort());
+
+ Snapshot good = snapshot("cluster0", 1234, "1");
+ cache.setSnapshot(CANARY, good);
+ cache.setSnapshot(STABLE, good);
+
+ // Give the Envoys time to connect, then roll out version 2.
+ Thread.sleep(10_000);
+ Snapshot candidate = snapshot("cluster1", 1235, "2");
+ cache.setSnapshot(CANARY, candidate);
+
+ AckTracker.Status status =
+ acks.await(CANARY, Resources.V3.CLUSTER_TYPE_URL, "2", 10, TimeUnit.SECONDS);
+ System.out.println("Canary answered version 2 with " + status);
+
+ if (status == AckTracker.Status.ACKED) {
+ // Here a real control plane would probe the canary before promoting.
+ cache.setSnapshot(STABLE, candidate);
+ System.out.println("Promoted version 2 to " + STABLE);
+ } else {
+ // A new version string, so the canary is sent the last good config again.
+ cache.setSnapshot(CANARY, snapshot("cluster0", 1234, "3"));
+ System.out.println("Rolled the canary back; " + STABLE + " never saw version 2");
+ }
+
+ server.awaitTermination();
+ }
+
+ private static Snapshot snapshot(String cluster, int port, String version) {
+ return Snapshot.create(
+ ImmutableList.of(
+ TestResources.createCluster(cluster, "127.0.0.1", port, Cluster.DiscoveryType.STATIC)),
+ ImmutableList.of(),
+ ImmutableList.of(),
+ ImmutableList.of(),
+ ImmutableList.of(),
+ version);
+ }
+}