Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,13 @@ mvn clean package
More thorough usage examples are still TODO, but there is a basic test implementation in
[TestMain](server/src/test/java/io/envoyproxy/controlplane/server/TestMain.java).

[CanaryRolloutExample](server/src/test/java/io/envoyproxy/controlplane/server/example/CanaryRolloutExample.java)
shows a staged rollout: one snapshot per group of Envoys (keyed by `node.cluster`), a new version
pushed to a canary group first, and promoted only after every canary Envoy has applied it.
[AckTracker](server/src/test/java/io/envoyproxy/controlplane/server/example/AckTracker.java), a
`DiscoveryServerCallbacks`, learns from the ACKs and NACKs in the xDS stream which version each
Envoy has applied, so the control plane can wait for "applied" rather than "pushed".

### Envoy API
There is automation built into this repo to keep the Envoy API (i.e. protobuf files)
up to date with the latest available Envoy release.
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
package io.envoyproxy.controlplane.server.example;

import io.envoyproxy.controlplane.server.DiscoveryServerCallbacks;
import io.envoyproxy.envoy.service.discovery.v3.DeltaDiscoveryRequest;
import io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest;
import io.envoyproxy.envoy.service.discovery.v3.DiscoveryResponse;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;

/**
* Learns which version of each resource type every connected Envoy has actually applied, from
* the xDS conversation itself.
*
* <p>In state-of-the-world xDS, Envoy answers every {@link DiscoveryResponse} with a new {@link
* DiscoveryRequest}. If {@code error_detail} is empty it is an ACK, and {@code version_info} is
* the version Envoy applied. If {@code error_detail} is set it is a NACK: Envoy rejected the
* response whose nonce it echoes, and keeps serving its previous config.
*
* <p>{@code SimpleCache.setSnapshot} only means "available"; waiting on this tracker means
* "applied". A control plane that rolls config out in stages (for example canary first) should
* wait for the ACK before judging or promoting the change. Register it with {@code new
* V3DiscoveryServer(tracker, cache)}.
*/
public class AckTracker implements DiscoveryServerCallbacks {

/** The outcome of waiting for a version. */
public enum Status {
/** Every connected Envoy in the group applied the version. */
ACKED,
/** At least one Envoy in the group rejected the version. */
NACKED,
/** Some Envoy has not answered yet. */
TIMEOUT,
/** No Envoy in the group is connected. */
NO_NODES
}

/** What one ADS stream (one Envoy) has told us. */
private static final class Stream {
volatile String group;
/** type URL -> last version ACKed / NACKed. */
final Map<String, String> acked = new ConcurrentHashMap<>();
final Map<String, String> nacked = new ConcurrentHashMap<>();
/** response nonce -> version sent with it, so a NACK can name the version it rejects. */
final Map<String, String> sent = new ConcurrentHashMap<>();
}

private final Map<Long, Stream> streams = new ConcurrentHashMap<>();
private final ReentrantLock lock = new ReentrantLock();
private final Condition changed = lock.newCondition();

@Override
public void onStreamOpen(long streamId, String typeUrl) {
streams.put(streamId, new Stream());
}

@Override
public void onStreamClose(long streamId, String typeUrl) {
streams.remove(streamId);
signal();
}

@Override
public void onStreamCloseWithError(long streamId, String typeUrl, Throwable error) {
streams.remove(streamId);
signal();
}

@Override
public void onV3StreamResponse(
long streamId, DiscoveryRequest request, DiscoveryResponse response) {
Stream s = streams.get(streamId);
if (s != null) {
s.sent.put(response.getNonce(), response.getVersionInfo());
}
}

@Override
public void onV3StreamRequest(long streamId, DiscoveryRequest request) {
Stream s = streams.computeIfAbsent(streamId, id -> new Stream());
if (request.hasNode()) {
s.group = request.getNode().getCluster();
}
if (request.getResponseNonce().isEmpty()) {
return; // the first request for a type is a subscription, not a reply
}
String sentVersion = s.sent.remove(request.getResponseNonce());
if (request.hasErrorDetail()) {
if (sentVersion != null) {
s.nacked.put(request.getTypeUrl(), sentVersion);
}
} else {
s.acked.put(request.getTypeUrl(), request.getVersionInfo());
}
signal();
}

@Override
public void onV3StreamDeltaRequest(long streamId, DeltaDiscoveryRequest request) {
// This example tracks state-of-the-world xDS only.
}

/**
* Waits until every Envoy connected in {@code group} has ACKed {@code version} of {@code
* typeUrl}, any of them NACKs it, or the timeout passes. Pass the latest version you pushed:
* versions are compared for equality, since snapshot versions are opaque strings.
*/
public Status await(String group, String typeUrl, String version, long timeout, TimeUnit unit)
throws InterruptedException {
long deadline = System.nanoTime() + unit.toNanos(timeout);
lock.lock();
try {
while (true) {
Status now = status(group, typeUrl, version);
if (now == Status.ACKED || now == Status.NACKED) {
return now;
}
long left = deadline - System.nanoTime();
if (left <= 0) {
return now;
}
changed.awaitNanos(left);
}
} finally {
lock.unlock();
}
}

private Status status(String group, String typeUrl, String version) {
boolean any = false;
boolean all = true;
for (Stream s : streams.values()) {
if (!group.equals(s.group)) {
continue;
}
any = true;
if (version.equals(s.nacked.get(typeUrl))) {
return Status.NACKED;
}
all &= version.equals(s.acked.get(typeUrl));
}
if (!any) {
return Status.NO_NODES;
}
return all ? Status.ACKED : Status.TIMEOUT;
}

private void signal() {
lock.lock();
try {
changed.signalAll();
} finally {
lock.unlock();
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
package io.envoyproxy.controlplane.server.example;

import static org.assertj.core.api.Assertions.assertThat;

import com.google.rpc.Status;
import io.envoyproxy.controlplane.cache.Resources;
import io.envoyproxy.envoy.config.core.v3.Node;
import io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest;
import io.envoyproxy.envoy.service.discovery.v3.DiscoveryResponse;
import java.util.concurrent.TimeUnit;
import org.junit.Test;

public class AckTrackerTest {

private static final String CLUSTERS = Resources.V3.CLUSTER_TYPE_URL;

private final AckTracker tracker = new AckTracker();

private void connect(long stream, String group) {
tracker.onStreamOpen(stream, "");
tracker.onV3StreamRequest(
stream,
DiscoveryRequest.newBuilder()
.setNode(Node.newBuilder().setId("envoy-" + stream).setCluster(group))
.setTypeUrl(CLUSTERS)
.build());
}

private void send(long stream, String version) {
tracker.onV3StreamResponse(
stream,
DiscoveryRequest.getDefaultInstance(),
DiscoveryResponse.newBuilder()
.setTypeUrl(CLUSTERS)
.setVersionInfo(version)
.setNonce("n" + version)
.build());
}

private void ack(long stream, String version) {
tracker.onV3StreamRequest(
stream,
DiscoveryRequest.newBuilder()
.setTypeUrl(CLUSTERS)
.setVersionInfo(version)
.setResponseNonce("n" + version)
.build());
}

private void nack(long stream, String version, String stillOn) {
tracker.onV3StreamRequest(
stream,
DiscoveryRequest.newBuilder()
.setTypeUrl(CLUSTERS)
.setVersionInfo(stillOn)
.setResponseNonce("n" + version)
.setErrorDetail(Status.newBuilder().setMessage("invalid cluster"))
.build());
}

private AckTracker.Status await(String group, String version) throws InterruptedException {
return tracker.await(group, CLUSTERS, version, 100, TimeUnit.MILLISECONDS);
}

@Test
public void ackedOnlyWhenEveryEnvoyInTheGroupAppliedTheVersion() throws Exception {
connect(1, "canary");
connect(2, "canary");
send(1, "2");
send(2, "2");
ack(1, "2");
assertThat(await("canary", "2")).isEqualTo(AckTracker.Status.TIMEOUT);
ack(2, "2");
assertThat(await("canary", "2")).isEqualTo(AckTracker.Status.ACKED);
}

@Test
public void nackedWhenAnyEnvoyRejectsTheVersion() throws Exception {
connect(1, "canary");
send(1, "2");
nack(1, "2", "1");
assertThat(await("canary", "2")).isEqualTo(AckTracker.Status.NACKED);
}

@Test
public void otherGroupsAndClosedStreamsDoNotCount() throws Exception {
connect(1, "stable");
send(1, "2");
ack(1, "2");
assertThat(await("canary", "2")).isEqualTo(AckTracker.Status.NO_NODES);
tracker.onStreamClose(1, "");
assertThat(await("stable", "2")).isEqualTo(AckTracker.Status.NO_NODES);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
package io.envoyproxy.controlplane.server.example;

import com.google.common.collect.ImmutableList;
import io.envoyproxy.controlplane.cache.Resources;
import io.envoyproxy.controlplane.cache.TestResources;
import io.envoyproxy.controlplane.cache.v3.SimpleCache;
import io.envoyproxy.controlplane.cache.v3.Snapshot;
import io.envoyproxy.controlplane.server.V3DiscoveryServer;
import io.envoyproxy.envoy.config.cluster.v3.Cluster;
import io.grpc.Server;
import io.grpc.netty.NettyServerBuilder;
import java.io.IOException;
import java.util.concurrent.TimeUnit;

/**
* Example: roll a config change out to a canary group of Envoys first, wait until every canary
* Envoy has applied it (ACK) or rejected it (NACK), and only then promote it to the rest.
*
* <p>Envoys are grouped by {@code node.cluster}, which Envoy sets from {@code --service-cluster}.
* Start one Envoy with {@code --service-cluster canary} and one with {@code --service-cluster
* stable}, both with an ADS bootstrap pointing at port 12345.
*
* <p>A real control plane would also check the canary's health (synthetic requests, or its error
* rate from metrics) after the ACK and before promoting. A NACK, a timeout or no connected canary
* all mean: do not promote, and put the canary back on the last good snapshot.
*/
public class CanaryRolloutExample {

private static final String CANARY = "canary";
private static final String STABLE = "stable";

/** Runs the example; the server stays up until the process is stopped. */
public static void main(String[] args) throws IOException, InterruptedException {
// One snapshot per group, chosen by the Envoy's node.cluster.
SimpleCache<String> cache = new SimpleCache<>(node -> node.getCluster());
AckTracker acks = new AckTracker();
V3DiscoveryServer discoveryServer = new V3DiscoveryServer(acks, cache);

Server server =
NettyServerBuilder.forPort(12345)
.addService(discoveryServer.getAggregatedDiscoveryServiceImpl())
.build()
.start();
Runtime.getRuntime().addShutdownHook(new Thread(server::shutdown));
System.out.println("xDS server listening on port " + server.getPort());

Snapshot good = snapshot("cluster0", 1234, "1");
cache.setSnapshot(CANARY, good);
cache.setSnapshot(STABLE, good);

// Give the Envoys time to connect, then roll out version 2.
Thread.sleep(10_000);
Snapshot candidate = snapshot("cluster1", 1235, "2");
cache.setSnapshot(CANARY, candidate);

AckTracker.Status status =
acks.await(CANARY, Resources.V3.CLUSTER_TYPE_URL, "2", 10, TimeUnit.SECONDS);
System.out.println("Canary answered version 2 with " + status);

if (status == AckTracker.Status.ACKED) {
// Here a real control plane would probe the canary before promoting.
cache.setSnapshot(STABLE, candidate);
System.out.println("Promoted version 2 to " + STABLE);
} else {
// A new version string, so the canary is sent the last good config again.
cache.setSnapshot(CANARY, snapshot("cluster0", 1234, "3"));
System.out.println("Rolled the canary back; " + STABLE + " never saw version 2");
}

server.awaitTermination();
}

private static Snapshot snapshot(String cluster, int port, String version) {
return Snapshot.create(
ImmutableList.of(
TestResources.createCluster(cluster, "127.0.0.1", port, Cluster.DiscoveryType.STATIC)),
ImmutableList.of(),
ImmutableList.of(),
ImmutableList.of(),
ImmutableList.of(),
version);
}
}