From 1d6aa4e2c8fd471d0696f7f4db1a1616a1e68672 Mon Sep 17 00:00:00 2001 From: Prachiti Talgulkar Date: Fri, 16 Oct 2026 17:33:09 +0530 Subject: [PATCH] e2e: add eks test case for scaling node Signed-off-by: Prachiti Talgulkar --- test/e2e/bootcnode_test.go | 176 +++++++++++++++++++++++++++++++++++++ 1 file changed, 176 insertions(+) diff --git a/test/e2e/bootcnode_test.go b/test/e2e/bootcnode_test.go index c83b982..51d053d 100644 --- a/test/e2e/bootcnode_test.go +++ b/test/e2e/bootcnode_test.go @@ -21,6 +21,7 @@ import ( meta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8stypes "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" bootcv1alpha1 "github.com/bootc-dev/bootc-operator/api/v1alpha1" @@ -1293,3 +1294,178 @@ func TestOperatorRestartResilience(t *testing.T) { Should(poolAllUpdated(1, env.NodeImageUpdateDigest())) t.Logf("Operator functioned normally after restart: rollout completed") } + +// TestMultiNodeRollout provisions a pool of multiple worker nodes +// (COS-5185: scale testing for bootc-operator) and verifies a rollout +// updates every node in the pool, respecting maxUnavailable, and that +// the pool status correctly reflects all nodes once the rollout +// completes. Unlike TestMidRolloutImageChange (bink-only, 2 nodes, +// exercises mid-rollout image switches), this test targets a larger +// pool and runs on EKS to validate behavior on real cloud nodes rather +// than the simulated bink provider, since upstream CI's GitHub runners +// cannot afford enough memory/CPU for multi-node scale testing. +func TestMultiNodeRollout(t *testing.T) { + e2eutil.Providers(t, "bink", "eks") + t.Parallel() + g := NewWithT(t) + g.SetDefaultEventuallyTimeout(pollTimeout) + g.SetDefaultEventuallyPollingInterval(pollInterval) + + env := e2eutil.New(t) + + const nodeCount = 3 + nodes := make([]string, nodeCount) + for i := range nodes { + nodes[i] = env.AddNode(t) + } + + ctx := context.Background() + + // Phase 1: Create pool with original image and wait for all nodes Idle. + // maxUnavailable bounds how many nodes may be disrupted at once. + const maxUnavailable = 1 + pool := env.NewPool("scale", env.NodeImageDigestedPullSpec(), + testutil.WithMaxUnavailable(intstr.FromInt(maxUnavailable))) + g.Expect(env.Client.Create(ctx, pool)).To(Succeed()) + + // Verify the daemon pod actually reaches Running on every node before + // waiting for Idle - a pod stuck in e.g. ImagePullBackOff would + // otherwise just show up as a confusing Idle-wait timeout below. + for _, nodeName := range nodes { + g.Eventually(func() ([]corev1.Pod, error) { + var pods corev1.PodList + err := env.Client.List(ctx, &pods, + client.InNamespace(testutil.OperatorNamespaceName), + client.MatchingFields{"spec.nodeName": nodeName}, + client.MatchingLabels{ + "app.kubernetes.io/name": "bootc-operator", + "app.kubernetes.io/component": "daemon", + }, + ) + return pods.Items, err + }).WithTimeout(3*time.Minute).Should(ConsistOf(And( + HaveField("Spec.NodeName", nodeName), + HaveField("Status.Phase", corev1.PodRunning), + )), "expected exactly one running daemon pod on %s", nodeName) + } + + for _, nodeName := range nodes { + testutil.WaitForNodeIdle(t, g, ctx, env.Client, nodeName, 3*time.Minute) + } + + t.Logf("All %d nodes are Idle with original image", nodeCount) + + g.Eventually(fetchPoolStatus(ctx, env.Client, pool)). + Should(poolAllUpdated(nodeCount, env.NodeImageDigest())) + + // Phase 2: Patch pool to the update image. + updateRef := env.NodeImageUpdateDigestedPullSpec() + + modified := pool.DeepCopy() + modified.Spec.Image.Ref = updateRef + g.Expect(env.Client.Patch(ctx, modified, client.MergeFrom(pool))).To(Succeed()) + *pool = *modified + + t.Logf("Patched pool to update image %s", updateRef) + + // Phase 3: Wait for every node to be Idle on the update digest - proves + // the rollout reached the full pool, not just a subset - while + // continuously enforcing maxUnavailable. Both checks share one polling + // loop so the invariant is sampled throughout the rollout rather than + // once before it starts, and so the wait ends as soon as the rollout + // does. Asserting the invariant once up front would pass trivially, + // since nothing is disrupted yet at that point. + // + // A node that stays Degraded aborts the wait early and reports the + // daemon's own error message. The daemon clears Degraded on the next + // successful reconcile, so a retried transient error must not fail the + // test; only a node degraded for longer than degradedGrace does. + const degradedGrace = 90 * time.Second + degradedSince := make(map[string]time.Time) + + g.Eventually(func(g Gomega) { + // Snapshot the whole pool in a single List. The assertions below + // abort the iteration at the first node that is behind, so the + // invariant has to be evaluated across every node before any of + // them run. It must also be one List rather than a Get per node: + // a List is a consistent snapshot at one resourceVersion, whereas + // sequential Gets can straddle a slot handoff and assemble a view + // in which two nodes hold a slot, a state that never existed. + var bnList bootcv1alpha1.BootcNodeList + g.Expect(env.Client.List(ctx, &bnList)).To(Succeed()) + + inPool := make(map[string]bool, len(nodes)) + for _, nodeName := range nodes { + inPool[nodeName] = true + } + bns := make([]bootcv1alpha1.BootcNode, 0, len(nodes)) + for i := range bnList.Items { + if inPool[bnList.Items[i].Name] { + bns = append(bns, bnList.Items[i]) + } + } + g.Expect(bns).To(HaveLen(len(nodes)), "expected a BootcNode per pool node") + + // maxUnavailable bounds reboot slots, not staging. Staging only + // downloads the image and disrupts nothing, so the controller lets + // the whole pool stage at once and gates cordon/drain/reboot. A + // node holds a slot exactly while it carries the in-reboot-slot + // annotation, which is also how the controller counts them. + // + // Exceeding the bound is a violated invariant rather than a state + // to wait out - retrying would let a later poll mask it. + inRebootSlot := 0 + for i := range bns { + if metav1.HasAnnotation(bns[i].ObjectMeta, bootcv1alpha1.AnnotationInRebootSlot) { + inRebootSlot++ + } + } + if inRebootSlot > maxUnavailable { + StopTrying(fmt.Sprintf( + "maxUnavailable=%d violated: %d nodes hold a reboot slot at once", + maxUnavailable, inRebootSlot, + )).Now() + } + + for i := range bns { + bn := &bns[i] + if cond := meta.FindStatusCondition( + bn.Status.Conditions, bootcv1alpha1.NodeDegraded, + ); cond != nil && cond.Status == metav1.ConditionTrue { + first, seen := degradedSince[bn.Name] + switch { + case !seen: + degradedSince[bn.Name] = time.Now() + case time.Since(first) > degradedGrace: + StopTrying(fmt.Sprintf( + "node %s degraded for over %s during rollout: %s: %s", + bn.Name, degradedGrace, cond.Reason, cond.Message, + )).Now() + } + } else { + delete(degradedSince, bn.Name) + } + } + + for i := range bns { + bn := &bns[i] + g.Expect(bn.Status.Conditions).To(ContainElement(And( + HaveField("Type", bootcv1alpha1.NodeIdle), + HaveField("Status", metav1.ConditionTrue), + HaveField("Reason", bootcv1alpha1.NodeReasonIdle), + )), "node %s is not Idle yet", bn.Name) + + g.Expect(bn.Status.Booted).NotTo(BeNil()) + g.Expect(bn.Status.Booted).To( + imageMatchesDigest(env.NodeImageUpdateDigest()), + "node %s has not booted the update digest yet", bn.Name, + ) + } + }).WithTimeout(20 * time.Minute).Should(Succeed()) + + t.Logf("All %d nodes are Idle with update image", nodeCount) + + // Phase 4: Verify final pool status reflects all nodes updated. + g.Eventually(fetchPoolStatus(ctx, env.Client, pool)). + Should(poolAllUpdated(nodeCount, env.NodeImageUpdateDigest())) +}