From 473cf94e9e6b80d649253609064195c049af0064 Mon Sep 17 00:00:00 2001 From: Zack Zlotnik Date: Fri, 9 Oct 2026 10:29:51 -0400 Subject: [PATCH] rollout: coordinate bootc updates with Karpenter Protect Karpenter-managed nodes from repair and voluntary disruption throughout image staging, reboot, and health verification, including their NodeClaims when available. Detect active Karpenter disruptions and reboots while deferring node drains and/or reboots until completion while preserving externally managed annotations and cordon state. Assisted-by: AI Signed-off-by: Zack Zlotnik --- api/v1alpha1/constants.go | 24 + config/rbac/role.yaml | 9 + docs/src/SUMMARY.md | 1 + docs/src/operations/karpenter.md | 195 +++ .../controller/bootcnodepool_controller.go | 178 ++- internal/controller/karpenter.go | 322 ++++ internal/controller/karpenter_test.go | 1406 +++++++++++++++++ internal/controller/rollout.go | 301 +++- .../rollout_karpenter_envtest_test.go | 256 +++ 9 files changed, 2650 insertions(+), 42 deletions(-) create mode 100644 docs/src/operations/karpenter.md create mode 100644 internal/controller/karpenter.go create mode 100644 internal/controller/karpenter_test.go create mode 100644 internal/controller/rollout_karpenter_envtest_test.go diff --git a/api/v1alpha1/constants.go b/api/v1alpha1/constants.go index d3d3338c..938eec8e 100644 --- a/api/v1alpha1/constants.go +++ b/api/v1alpha1/constants.go @@ -30,4 +30,28 @@ const ( // state for which the controller considered emitting an event. It is // observability bookkeeping only and is never used to drive reconciliation. AnnotationLastObservedState = "bootc.dev/last-observed-state" + + // AnnotationOwnsKarpenterDoNotRepair is set on a BootcNode to record that + // the controller added the karpenter.sh/do-not-repair annotation to the K8s + // Node during slot assignment. When present, the controller removes the + // karpenter.sh/do-not-repair annotation from the Node when the slot is freed. + // It is not set when the annotation was already present before slot assignment + // (externally set), so the controller never removes an annotation it did not add. + AnnotationOwnsKarpenterDoNotRepair = "bootc.dev/owns-karpenter-do-not-repair" + + // AnnotationOwnsKarpenterDoNotDisrupt is set on a BootcNode to record that + // the controller added the karpenter.sh/do-not-disrupt annotation to the K8s + // Node during slot assignment. When present, the controller removes the + // karpenter.sh/do-not-disrupt annotation from the Node when the slot is freed. + // It is not set when the annotation was already present before slot assignment + // (externally set), so the controller never removes an annotation it did not add. + AnnotationOwnsKarpenterDoNotDisrupt = "bootc.dev/owns-karpenter-do-not-disrupt" + + // AnnotationOwnsKarpenterNodeClaimDoNotRepair records that the controller + // added do-not-repair to the associated Karpenter NodeClaim. + AnnotationOwnsKarpenterNodeClaimDoNotRepair = "bootc.dev/owns-karpenter-nodeclaim-do-not-repair" + + // AnnotationOwnsKarpenterNodeClaimDoNotDisrupt records that the controller + // added do-not-disrupt to the associated Karpenter NodeClaim. + AnnotationOwnsKarpenterNodeClaimDoNotDisrupt = "bootc.dev/owns-karpenter-nodeclaim-do-not-disrupt" ) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index c25de146..28818c10 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -49,6 +49,15 @@ rules: - daemonsets verbs: - get +- apiGroups: + - karpenter.sh + resources: + - nodeclaims + verbs: + - get + - list + - patch + - watch - apiGroups: - node.bootc.dev resources: diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index fbfaf433..d411f43b 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -8,3 +8,4 @@ - [Helm Chart](helm.md) - [Operations](operations/index.md) - [Managing a pool](operations/pool.md) + - [Karpenter integration](operations/karpenter.md) diff --git a/docs/src/operations/karpenter.md b/docs/src/operations/karpenter.md new file mode 100644 index 00000000..b066486a --- /dev/null +++ b/docs/src/operations/karpenter.md @@ -0,0 +1,195 @@ +# Karpenter integration + +bootc-operator is aware of [Karpenter](https://karpenter.sh/) and coordinates +with it during node updates to prevent unnecessary node replacement, +voluntary disruption, and concurrent reboot handling. + +## Why this matters + +Karpenter monitors nodes for health and may decide to replace or disrupt a +node that appears unhealthy or underutilized — for example, one that is +cordoned, being drained, or not yet Ready after a reboot. All of those +conditions arise naturally during a bootc-operator update window, so without +coordination Karpenter could race with the operator and terminate a node that +is in the middle of a legitimate OS update. + +Karpenter respects two node-level annotations: + +- `karpenter.sh/do-not-repair: "true"` — prevents Node Auto Repair from + replacing a node that has unhealthy conditions. +- `karpenter.sh/do-not-disrupt: "true"` — prevents voluntary disruption + actions (consolidation, drift) from selecting this node. + +> **Note:** `karpenter.sh/do-not-repair` was introduced in Karpenter v1.15.0. +> That is the minimum required version for full bootc-operator Karpenter integration. + +bootc-operator sets both annotations automatically on Karpenter-managed nodes +for the duration of the full update lifecycle — from image staging through +node reboot and health verification. Where the associated `NodeClaim` exists, +it also sets the annotations there. This is best-effort coordination, not a +distributed lock. + +## How it works + +When a Karpenter-owned node enters the `Staging` state (the daemon is pulling +the new image), bootc-operator immediately sets `do-not-repair` and +`do-not-disrupt` on the Node and, where available, the associated NodeClaim. +This protection persists through slot assignment, drain, reboot, and post-boot +health verification. + +Before assigning a reboot slot or starting a drain, the controller checks +whether Karpenter has already committed any lifecycle operation that would +conflict with the bootc update. It checks: + +- Node taints `karpenter.sh/rebooting:NoSchedule` and + `karpenter.sh/disrupted:NoSchedule` +- Node `DeletionTimestamp` (node is being terminated) +- NodeClaim `DeletionTimestamp` (NodeClaim is being deleted) +- NodeClaim `status.conditions`: `Rebooting=True`, `DisruptionReason=True`, + or `InstanceTerminating=True` + +If any of these signals is active, bootc-operator defers and retries. It +never removes a Karpenter taint. + +If the Karpenter NodeClaim owner reference is present but the NodeClaim object +returns a 404, bootc-operator treats this as unsafe (the NodeClaim may be +mid-deletion) and defers with a short requeue rather than proceeding. + +If none of those signals is active, the controller takes the following steps: + +1. For each annotation (`do-not-repair`, `do-not-disrupt`): if the annotation + is **not** already set to `"true"`, the controller sets it (value `"true"`) + on the Node and, where available, the NodeClaim. It records ownership + separately on the `BootcNode` object: + - `bootc.dev/owns-karpenter-do-not-repair: "true"` + - `bootc.dev/owns-karpenter-do-not-disrupt: "true"` + - `bootc.dev/owns-karpenter-nodeclaim-do-not-repair: "true"` + - `bootc.dev/owns-karpenter-nodeclaim-do-not-disrupt: "true"` +2. The controller proceeds through the normal update flow: drain, reboot, and + image verification. +3. When the slot is freed (node is Ready and running the new image), the + controller removes only annotations that it still owns and whose value is + still `"true"`. It never removes annotations it did not add. + +## NodeClaim annotations are supplemental + +NodeClaim annotations (`do-not-repair`, `do-not-disrupt`) are a supplemental +signal, not a distributed lock. By the time bootc-operator writes them, +Karpenter may already have selected or queued a disruption for the registered +Node. The authoritative disruption checks are the Node-level taint, the +NodeClaim deletion timestamp, and the NodeClaim status conditions — the +controller checks these before both slot assignment and drain. + +## NodeClaim watch registration + +When the controller starts, it checks whether the Karpenter `NodeClaim` API is +available. If it is, the controller registers a watch for NodeClaim events and +uses changes to the associated NodeClaim to requeue its owning +`BootcNodePool`. This allows changes such as Karpenter starting or completing +a reboot to be observed promptly. + +The watch is registered only during startup. If Karpenter is installed after +bootc-operator has started, the operator does not add the watch dynamically; +restart the bootc-operator controller after installing Karpenter. As a fallback, +the controller schedules a periodic requeue (every 30 seconds) whenever +Karpenter-owned nodes are active, so that missed NodeClaim events do not +stall progress indefinitely. + +The controller represents NodeClaims with Kubernetes `Unstructured` objects +instead of importing Karpenter's Go API types. This keeps Karpenter an optional +integration and avoids adding a Karpenter API dependency or scheme registration +to bootc-operator. It also lets clusters without the NodeClaim CRD continue to +run normally. The tradeoff is that the small set of fields used here, such as +`status.nodeName` and `status.conditions`, is checked at runtime rather than +being enforced by Go types. + +## Pre-existing annotations + +If a Karpenter annotation is already present on the Node with a value of +`"true"` before protection is applied — set by an external actor, another +controller, or a manual operation — bootc-operator leaves it alone: + +- It does **not** set the corresponding tracking annotation on the `BootcNode`. +- It does **not** remove the annotation when the slot is freed. + +If the annotation is present but set to any value other than `"true"` (for +example `"false"`), bootc-operator overwrites it with `"true"` and records +ownership. This is required because Karpenter only honors the literal value +`"true"`; any other value leaves the node unprotected. + +The two annotations are tracked independently. If only one is already set to +`"true"`, the operator owns (and later removes) only the other. + +## Temporary overprovisioning during node drains + +During a bootc-operator update, the following sequence can cause temporary +overprovisioning: + +1. bootc-operator evicts workloads from the node being updated. +2. The evicted workloads become pending. +3. Karpenter provisions new capacity to satisfy the pending pods. +4. The original node reboots into the new image and rejoins the cluster. +5. The newly provisioned capacity may become excess and be consolidated later. + +This is expected behavior. The `do-not-disrupt` annotation suppresses +Karpenter's voluntary disruption of the node being updated, but it does not +suppress Karpenter provisioning new capacity for pending workloads. Karpenter +provisioning is driven by pending pod demand rather than disruption eligibility. + +## Limitations + +`karpenter.sh/do-not-disrupt` only blocks voluntary disruption methods +(consolidation and drift). It does not protect a node against forceful +termination triggered by expiration (`expireAfter`), interruption (spot +instance reclaim), Node Auto Repair, or manual deletion. If any of those +events occur while a node is in a bootc-operator reboot slot, Karpenter may +still terminate it. + +`karpenter.sh/do-not-repair` similarly has no effect on voluntary disruption +or expiration — it only suppresses Node Auto Repair. + +If a node is terminated by expiration, interruption, or any other forceful +method while a bootc-operator update is in progress, the termination is caused +by Karpenter acting on its own policy — not by the bootc-operator update. The +update simply happened to be in flight at the time. + +## Residual race and coordination guarantees + +The annotations and disruption checks are not an authoritative mutex. Karpenter +does not currently acknowledge `do-not-repair`, expose a shared Lease, or +coordinate its reboot commit with bootc-operator's slot assignment. Karpenter +can therefore commit a disruption immediately before or after bootc-operator's +last check. Setting `do-not-repair` does not cancel a disruption that Karpenter +has already committed. + +The integration narrows this window substantially by: + +- Applying protection from the start of image staging rather than only at + slot assignment. +- Checking all disruption signals (reboot taint, disrupted taint, deletion + timestamp, and NodeClaim conditions) before both slot assignment and drain. + +Full serialization requires Karpenter support for a shared ownership protocol +or an admission/provider mechanism that prevents a competing operation from +being committed. + +Refer to the [Karpenter disruption documentation](https://karpenter.sh/docs/concepts/disruption/) +for a full breakdown of which controls apply to which disruption methods. + +## Non-Karpenter nodes + +Nodes without a `NodeClaim` owner reference are unaffected. The feature +activates only when the owner reference check matches. + +## Annotation reference + +| Annotation | Object | Managed by | Purpose | +|---|---|---|---| +| `karpenter.sh/do-not-repair` | K8s Node | bootc-operator (or external) | Prevents Karpenter Node Auto Repair from replacing this node during an update | +| `karpenter.sh/do-not-disrupt` | K8s Node | bootc-operator (or external) | Prevents Karpenter voluntary disruption (consolidation, drift) during an update | +| `karpenter.sh/do-not-repair` | NodeClaim | bootc-operator (or external) | Best-effort suppression of Karpenter repair selection before an operator reboot | +| `karpenter.sh/do-not-disrupt` | NodeClaim | bootc-operator (or external) | Best-effort suppression of Karpenter voluntary disruption before an operator reboot | +| `bootc.dev/owns-karpenter-do-not-repair` | BootcNode | bootc-operator | Tracks that the operator added `do-not-repair`; cleared when the annotation is removed on slot free | +| `bootc.dev/owns-karpenter-do-not-disrupt` | BootcNode | bootc-operator | Tracks that the operator added `do-not-disrupt`; cleared when the annotation is removed on slot free | +| `bootc.dev/owns-karpenter-nodeclaim-do-not-repair` | BootcNode | bootc-operator | Tracks ownership of NodeClaim `do-not-repair` | +| `bootc.dev/owns-karpenter-nodeclaim-do-not-disrupt` | BootcNode | bootc-operator | Tracks ownership of NodeClaim `do-not-disrupt` | diff --git a/internal/controller/bootcnodepool_controller.go b/internal/controller/bootcnodepool_controller.go index 787e654b..426ba8ea 100644 --- a/internal/controller/bootcnodepool_controller.go +++ b/internal/controller/bootcnodepool_controller.go @@ -18,8 +18,10 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/events" @@ -89,6 +91,7 @@ type BootcNodePoolReconciler struct { // +kubebuilder:rbac:groups=apps,resources=daemonsets,verbs=get // +kubebuilder:rbac:groups="",resources=events,verbs=create;patch // +kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=create;patch +// +kubebuilder:rbac:groups=karpenter.sh,resources=nodeclaims,verbs=get;list;watch;patch // SetupWithManager sets up the controller with the Manager. func (r *BootcNodePoolReconciler) SetupWithManager(mgr ctrl.Manager) error { @@ -96,14 +99,59 @@ func (r *BootcNodePoolReconciler) SetupWithManager(mgr ctrl.Manager) error { r.drainCh = make(chan event.GenericEvent, 100) r.drains = make(map[string]*drainStatus) - return ctrl.NewControllerManagedBy(mgr). + // ctrlBuilder avoids shadowing the imported "sigs.k8s.io/controller-runtime/pkg/builder" package. + ctrlBuilder := ctrl.NewControllerManagedBy(mgr). For(&bootcv1alpha1.BootcNodePool{}). Owns(&bootcv1alpha1.BootcNode{}). Watches(&corev1.Node{}, handler.EnqueueRequestsFromMapFunc(r.mapNodeToPoolRequests), builder.WithPredicates(nodePredicates())). Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(r.mapSecretToPoolRequests), builder.WithPredicates(secretTypePredicate())). - WatchesRawSource(source.Channel(r.drainCh, &handler.EnqueueRequestForObject{})). - Named("bootcnodepool"). - Complete(r) + WatchesRawSource(source.Channel(r.drainCh, &handler.EnqueueRequestForObject{})) + + // Karpenter is optional. Only register the NodeClaim watch when its CRD is + // discoverable, so the operator remains usable on ordinary clusters. + if _, err := mgr.GetRESTMapper().RESTMapping( + schema.GroupKind{Group: "karpenter.sh", Kind: karpenterNodeClaimKind}, + "v1", + ); err == nil { + ctrlBuilder = ctrlBuilder.Watches( + &unstructured.Unstructured{Object: map[string]interface{}{ + "apiVersion": karpenterNodeClaimAPIVersion, + "kind": karpenterNodeClaimKind, + }}, + handler.EnqueueRequestsFromMapFunc(r.mapNodeClaimToPoolRequests), + ) + } + + return ctrlBuilder.Named("bootcnodepool").Complete(r) +} + +// mapNodeClaimToPoolRequests maps a Karpenter NodeClaim status update to the +// BootcNodePool owning the corresponding Node. +func (r *BootcNodePoolReconciler) mapNodeClaimToPoolRequests( + ctx context.Context, + obj client.Object, +) []reconcile.Request { + log := logf.FromContext(ctx) + claim, ok := obj.(*unstructured.Unstructured) + if !ok { + return nil + } + nodeName, found, err := unstructured.NestedString(claim.Object, "status", "nodeName") + if err != nil || !found || nodeName == "" { + return nil + } + var bn bootcv1alpha1.BootcNode + if err := r.Get(ctx, types.NamespacedName{Name: nodeName}, &bn); err != nil { + if !apierrors.IsNotFound(err) { + log.Error(err, "Failed to get BootcNode in NodeClaim mapper", "node", nodeName) + } + return nil + } + owner := metav1.GetControllerOf(&bn) + if owner == nil || owner.Kind != "BootcNodePool" { + return nil + } + return []reconcile.Request{{NamespacedName: types.NamespacedName{Name: owner.Name}}} } // mapNodeToPoolRequests maps a Node event to the BootcNodePool(s) that should @@ -213,8 +261,8 @@ func nodeSelectorMatchesNode(sel *metav1.LabelSelector, node *corev1.Node) (bool } // nodePredicates returns predicates that filter Node events to only those -// relevant to pool membership: label changes, Ready condition changes, and -// spec.unschedulable changes. +// relevant to pool membership or reboot coordination: label changes, Ready +// condition changes, spec.unschedulable changes, and the Karpenter reboot taint. func nodePredicates() predicate.Predicate { return predicate.Funcs{ CreateFunc: func(e event.CreateEvent) bool { @@ -231,7 +279,8 @@ func nodePredicates() predicate.Predicate { } return nodeLabelsChanged(oldNode, newNode) || nodeReadyConditionChanged(oldNode, newNode) || - nodeUnschedulableChanged(oldNode, newNode) + nodeUnschedulableChanged(oldNode, newNode) || + karpenterRebootTaintChanged(oldNode, newNode) }, GenericFunc: func(e event.GenericEvent) bool { return true @@ -260,6 +309,20 @@ func nodeUnschedulableChanged(oldNode, newNode *corev1.Node) bool { return oldNode.Spec.Unschedulable != newNode.Spec.Unschedulable } +func karpenterRebootTaintChanged(oldNode, newNode *corev1.Node) bool { + return hasKarpenterRebootTaint(oldNode) != hasKarpenterRebootTaint(newNode) +} + +func hasKarpenterRebootTaint(node *corev1.Node) bool { + for _, taint := range node.Spec.Taints { + if taint.Key == karpenterRebootingTaintKey && + taint.Effect == karpenterRebootingTaintEffect { + return true + } + } + return false +} + // secretTypePredicate returns a predicate that accepts only Secrets of // type kubernetes.io/dockerconfigjson. func secretTypePredicate() predicate.Predicate { @@ -859,8 +922,8 @@ func (r *BootcNodePoolReconciler) removeBootcNode( if err := r.Get(ctx, types.NamespacedName{Name: bn.Name}, &node); err == nil { // No point in cleaning up the Node object if it's going away anyway... if node.DeletionTimestamp == nil { - if err := r.restoreCordonState(ctx, bn, &node); err != nil { - return fmt.Errorf("restoring cordon state: %w", err) + if err := r.restoreNodeState(ctx, bn, &node); err != nil { + return fmt.Errorf("restoring node state: %w", err) } if err := r.ensureManagedLabel(ctx, &node, false); err != nil { return fmt.Errorf("removing managed label: %w", err) @@ -877,23 +940,100 @@ func (r *BootcNodePoolReconciler) removeBootcNode( return nil } -// restoreCordonState uncordons the Node if the BootcNode's was-cordoned -// annotation indicates the operator cordoned it. If the annotation is absent -// or "true" (node was already cordoned before us), this is a no-op. -func (r *BootcNodePoolReconciler) restoreCordonState( +// restoreNodeState removes any Karpenter annotations the controller owns and +// restores the node's cordon state, combining both into a single patch when +// needed. Karpenter annotation ownership is tracked via BootcNode annotations +// rather than inspecting the Node directly, so an external actor removing them +// doesn't cause spurious patches. The cordon is only removed if the was-cordoned +// annotation records that the operator was the one who cordoned it. +func (r *BootcNodePoolReconciler) restoreNodeState( ctx context.Context, bn *bootcv1alpha1.BootcNode, node *corev1.Node, ) error { - if bn.Annotations[bootcv1alpha1.AnnotationWasCordoned] != "false" { + log := logf.FromContext(ctx) + modified := node.DeepCopy() + changed := false + + if metav1.HasAnnotation(bn.ObjectMeta, bootcv1alpha1.AnnotationOwnsKarpenterDoNotRepair) { + if modified.Annotations[karpenterDoNotRepairAnnotationKey] == "true" { + log.Info("Removing karpenter.sh/do-not-repair from node", "node", node.Name) + delete(modified.Annotations, karpenterDoNotRepairAnnotationKey) + changed = true + } + } + + if metav1.HasAnnotation(bn.ObjectMeta, bootcv1alpha1.AnnotationOwnsKarpenterDoNotDisrupt) { + if modified.Annotations[karpenterDoNotDisruptAnnotationKey] == "true" { + log.Info("Removing karpenter.sh/do-not-disrupt from node", "node", node.Name) + delete(modified.Annotations, karpenterDoNotDisruptAnnotationKey) + changed = true + } + } + + if metav1.HasAnnotation(bn.ObjectMeta, bootcv1alpha1.AnnotationWasCordoned) && + bn.Annotations[bootcv1alpha1.AnnotationWasCordoned] == "false" { + log.Info("Uncordoning node", "node", node.Name) + modified.Spec.Unschedulable = false + changed = true + } + + if changed { + if err := r.Patch(ctx, modified, client.StrategicMergeFrom(node)); err != nil { + return fmt.Errorf("restoring node state: %w", err) + } + *node = *modified + } + + // Only fetch the NodeClaim when we own at least one of its annotations, + // to avoid a spurious API call for non-Karpenter nodes. + ownsNodeClaimRepair := metav1.HasAnnotation( + bn.ObjectMeta, + bootcv1alpha1.AnnotationOwnsKarpenterNodeClaimDoNotRepair, + ) + ownsNodeClaimDisrupt := metav1.HasAnnotation( + bn.ObjectMeta, + bootcv1alpha1.AnnotationOwnsKarpenterNodeClaimDoNotDisrupt, + ) + + // If we don't own either of these annotations, then there is nothing to do. + if !ownsNodeClaimRepair && !ownsNodeClaimDisrupt { return nil } - modified := node.DeepCopy() - modified.Spec.Unschedulable = false - if err := r.Patch(ctx, modified, client.StrategicMergeFrom(node)); err != nil { - return fmt.Errorf("uncordoning node: %w", err) + + claim, _, err := r.getKarpenterNodeClaim(ctx, node) + if err != nil { + return fmt.Errorf("getting Karpenter NodeClaim: %w", err) } - *node = *modified + + // If the NodeClaim is absent (deleted or CRD missing), skip the patch. + // The node may have been forcefully replaced by Karpenter; there is nothing + // to clean up on a NodeClaim that no longer exists. + if claim == nil { + return nil + } + + claimModified := claim.DeepCopy() + claimAnnotations := claimModified.GetAnnotations() + claimChanged := false + + if ownsNodeClaimRepair && claimAnnotations[karpenterDoNotRepairAnnotationKey] == "true" { + delete(claimAnnotations, karpenterDoNotRepairAnnotationKey) + claimChanged = true + } + + if ownsNodeClaimDisrupt && claimAnnotations[karpenterDoNotDisruptAnnotationKey] == "true" { + delete(claimAnnotations, karpenterDoNotDisruptAnnotationKey) + claimChanged = true + } + + if claimChanged { + claimModified.SetAnnotations(claimAnnotations) + if err := r.Patch(ctx, claimModified, client.MergeFrom(claim)); err != nil { + return fmt.Errorf("restoring Karpenter NodeClaim annotations: %w", err) + } + } + return nil } diff --git a/internal/controller/karpenter.go b/internal/controller/karpenter.go new file mode 100644 index 00000000..980f12bd --- /dev/null +++ b/internal/controller/karpenter.go @@ -0,0 +1,322 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +// karpenter.go contains all Karpenter-specific controller logic: constants, +// types, and methods that coordinate bootc-operator node updates with Karpenter +// lifecycle management. The integration is designed to be optional — clusters +// without the Karpenter NodeClaim CRD continue to reconcile normally. +// +// See docs/src/operations/karpenter.md for a full description of the +// integration design, annotation semantics, and known limitations. + +import ( + "context" + "fmt" + "maps" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + bootcv1alpha1 "github.com/bootc-dev/bootc-operator/api/v1alpha1" +) + +// Karpenter integration constants. The controller checks for a NodeClaim owner +// reference and manages karpenter.sh/do-not-repair and +// karpenter.sh/do-not-disrupt to prevent Karpenter from replacing or +// voluntarily disrupting a node while it is being updated. +// See docs/src/operations/karpenter.md. +const ( + karpenterDoNotRepairAnnotationKey = "karpenter.sh/do-not-repair" + karpenterDoNotDisruptAnnotationKey = "karpenter.sh/do-not-disrupt" + karpenterRebootingTaintKey = "karpenter.sh/rebooting" + karpenterDisruptedTaintKey = "karpenter.sh/disrupted" + karpenterRebootingTaintEffect = corev1.TaintEffectNoSchedule + karpenterNodeClaimAPIVersion = "karpenter.sh/v1" + karpenterNodeClaimKind = "NodeClaim" + + // karpenterMissingClaimRequeue is how long driveRollout waits before + // retrying when a Karpenter-owned node's NodeClaim is unexpectedly + // absent. The NodeClaim may be mid-deletion; waiting avoids treating a + // node under Karpenter termination as safe to update. + karpenterMissingClaimRequeue = 30 * time.Second +) + +// karpenterNodeClaimResult is the outcome of a NodeClaim lookup. +type karpenterNodeClaimResult int + +const ( + // karpenterClaimFound means the NodeClaim exists and was returned. + karpenterClaimFound karpenterNodeClaimResult = iota + // karpenterClaimAbsent means the node has a NodeClaim owner reference but + // the object returned 404. The NodeClaim may be mid-deletion; callers + // should treat this as unsafe and defer. + karpenterClaimAbsent + // karpenterClaimCRDMissing means the Karpenter NodeClaim CRD is not + // installed on this cluster. Treat the node as non-Karpenter. + karpenterClaimCRDMissing + // karpenterClaimNoRef means the node has no Karpenter NodeClaim owner + // reference at all. + karpenterClaimNoRef + // karpenterClaimError means the API call failed for a transient or + // unexpected reason. The caller should propagate the accompanying error. + karpenterClaimError +) + +// isNodeOwnedByKarpenterNodeClaim reports whether the K8s Node has an owner +// reference pointing to a Karpenter NodeClaim (karpenter.sh/v1). +func isNodeOwnedByKarpenterNodeClaim(node *corev1.Node) bool { + for _, ref := range node.OwnerReferences { + if ref.APIVersion == karpenterNodeClaimAPIVersion && ref.Kind == karpenterNodeClaimKind { + return true + } + } + return false +} + +// getKarpenterNodeClaim returns the NodeClaim owning node, if one is available. +// The unstructured type keeps Karpenter optional; clusters without its CRD +// continue to reconcile normally. +// +// The second return value distinguishes five cases: +// - karpenterClaimFound: claim is non-nil, everything is normal. +// - karpenterClaimAbsent: the node has a NodeClaim owner ref but the object +// returned 404. This is unsafe — the NodeClaim may be mid-deletion. +// - karpenterClaimCRDMissing: the Karpenter CRD is not installed; treat the +// node as non-Karpenter. +// - karpenterClaimNoRef: no Karpenter owner reference on this node. +// - karpenterClaimError: a transient or unexpected API error; the +// accompanying error value is non-nil and should be propagated. +func (r *BootcNodePoolReconciler) getKarpenterNodeClaim( + ctx context.Context, + node *corev1.Node, +) (*unstructured.Unstructured, karpenterNodeClaimResult, error) { + for _, ref := range node.OwnerReferences { + if ref.APIVersion != karpenterNodeClaimAPIVersion || ref.Kind != karpenterNodeClaimKind { + continue + } + claim := &unstructured.Unstructured{} + claim.SetAPIVersion(karpenterNodeClaimAPIVersion) + claim.SetKind(karpenterNodeClaimKind) + if err := r.Get(ctx, types.NamespacedName{Name: ref.Name}, claim); err != nil { + if apimeta.IsNoMatchError(err) { + // CRD not installed — Karpenter is not present on this cluster. + return nil, karpenterClaimCRDMissing, nil + } + if apierrors.IsNotFound(err) { + // Owner ref points to a NodeClaim that no longer exists. + // This is unsafe: Karpenter may be replacing or terminating + // the node right now. + return nil, karpenterClaimAbsent, nil + } + return nil, karpenterClaimError, err + } + return claim, karpenterClaimFound, nil + } + return nil, karpenterClaimNoRef, nil +} + +// isKarpenterDisruptionActive reports whether Karpenter currently owns any +// lifecycle operation that would conflict with a bootc-operator update. It +// checks for: +// +// - Node or NodeClaim deletion in progress (DeletionTimestamp set) +// - Karpenter reboot taint on the Node +// - Karpenter disrupted taint on the Node (voluntary disruption committed) +// - NodeClaim status conditions: Rebooting, DisruptionReason, or +// InstanceTerminating set to True +// +// It also handles the case where the NodeClaim owner reference is present but +// the object is absent (mid-deletion); in that case it returns +// (true, true, nil, nil) to signal "defer without error". The second return +// value is true when the NodeClaim is missing and the caller should arrange a +// requeue rather than continuing immediately. +// +// When active is false and a NodeClaim was successfully fetched, claim is +// non-nil. Callers that need the NodeClaim (e.g. assignRebootSlot) can reuse +// it to avoid a second API round-trip. +func (r *BootcNodePoolReconciler) isKarpenterDisruptionActive( + ctx context.Context, + node *corev1.Node, +) (active bool, claimMissing bool, claim *unstructured.Unstructured, err error) { + // A deleting Node is already being terminated — always defer. + if node.DeletionTimestamp != nil { + return true, false, nil, nil + } + + // Fast path: check Node taints before making any API calls. + for _, taint := range node.Spec.Taints { + if taint.Effect != karpenterRebootingTaintEffect { + continue + } + if taint.Key == karpenterRebootingTaintKey || taint.Key == karpenterDisruptedTaintKey { + return true, false, nil, nil + } + } + + claim, result, err := r.getKarpenterNodeClaim(ctx, node) + if err != nil { + return false, false, nil, err + } + switch result { + case karpenterClaimNoRef, karpenterClaimCRDMissing: + // Not a Karpenter node, or Karpenter not installed. + return false, false, nil, nil + case karpenterClaimAbsent: + // NodeClaim owner ref exists but object is missing. This is unsafe: + // Karpenter may be mid-termination. Signal the caller to requeue. + return true, true, nil, nil + } + + // The NodeClaim exists. Check for a pending deletion first. + if claim.GetDeletionTimestamp() != nil { + return true, false, nil, nil + } + + // Check NodeClaim status conditions. + conditions, found, err := unstructured.NestedSlice(claim.Object, "status", "conditions") + if err != nil || !found { + return false, false, nil, err + } + for _, item := range conditions { + condition, ok := item.(map[string]interface{}) + if !ok || condition["status"] != "True" { + continue + } + switch condition["type"] { + case "Rebooting", "DisruptionReason", "InstanceTerminating": + return true, false, nil, nil + } + } + return false, false, claim, nil +} + +// ensureKarpenterStagingProtection applies Karpenter do-not-repair and +// do-not-disrupt annotations to a Karpenter-owned node that has entered the +// Staging state but has not yet been assigned a reboot slot. This extends the +// protection window to cover the full update lifecycle (image pull → staged → +// reboot → healthy), rather than starting only at slot assignment. +// +// It reuses the same ownership-tracking annotations as assignRebootSlot, so +// restoreNodeState/freeRebootSlot will clean them up at slot free time. +func (r *BootcNodePoolReconciler) ensureKarpenterStagingProtection( + ctx context.Context, + bn *bootcv1alpha1.BootcNode, + node *corev1.Node, +) error { + if !isNodeOwnedByKarpenterNodeClaim(node) { + return nil + } + log := logf.FromContext(ctx) + + claim, claimResult, err := r.getKarpenterNodeClaim(ctx, node) + if err != nil { + return fmt.Errorf("getting NodeClaim for staging protection: %w", err) + } + claimAvailable := claimResult == karpenterClaimFound + + // Build desired BootcNode state. + modifiedBN := bn.DeepCopy() + if node.Annotations[karpenterDoNotRepairAnnotationKey] != "true" { + metav1.SetMetaDataAnnotation( + &modifiedBN.ObjectMeta, + bootcv1alpha1.AnnotationOwnsKarpenterDoNotRepair, + "true", + ) + } + if node.Annotations[karpenterDoNotDisruptAnnotationKey] != "true" { + metav1.SetMetaDataAnnotation( + &modifiedBN.ObjectMeta, + bootcv1alpha1.AnnotationOwnsKarpenterDoNotDisrupt, + "true", + ) + } + if claimAvailable { + claimAnnotations := claim.GetAnnotations() + if claimAnnotations[karpenterDoNotRepairAnnotationKey] != "true" { + metav1.SetMetaDataAnnotation( + &modifiedBN.ObjectMeta, + bootcv1alpha1.AnnotationOwnsKarpenterNodeClaimDoNotRepair, + "true", + ) + } + if claimAnnotations[karpenterDoNotDisruptAnnotationKey] != "true" { + metav1.SetMetaDataAnnotation( + &modifiedBN.ObjectMeta, + bootcv1alpha1.AnnotationOwnsKarpenterNodeClaimDoNotDisrupt, + "true", + ) + } + } + + if !annotationsEqual(bn.Annotations, modifiedBN.Annotations) { + log.Info("Applying Karpenter staging protection", "node", bn.Name) + if err := r.Patch(ctx, modifiedBN, client.MergeFrom(bn)); err != nil { + return fmt.Errorf("annotating BootcNode for staging protection: %w", err) + } + *bn = *modifiedBN + } + + // Apply fence to NodeClaim. + if claimAvailable { + claimAnnotations := claim.GetAnnotations() + modifiedClaimAnnotations := map[string]string{} + maps.Copy(modifiedClaimAnnotations, claimAnnotations) + if claimAnnotations[karpenterDoNotRepairAnnotationKey] != "true" { + modifiedClaimAnnotations[karpenterDoNotRepairAnnotationKey] = "true" + } + if claimAnnotations[karpenterDoNotDisruptAnnotationKey] != "true" { + modifiedClaimAnnotations[karpenterDoNotDisruptAnnotationKey] = "true" + } + if !annotationsEqual(claimAnnotations, modifiedClaimAnnotations) { + modifiedClaim := claim.DeepCopy() + modifiedClaim.SetAnnotations(modifiedClaimAnnotations) + if err := r.Patch(ctx, modifiedClaim, client.MergeFrom(claim)); err != nil { + return fmt.Errorf("patching NodeClaim for staging protection: %w", err) + } + } + } + + // Apply fence to Node. + modifiedNode := node.DeepCopy() + if node.Annotations[karpenterDoNotRepairAnnotationKey] != "true" { + log.Info("Setting karpenter.sh/do-not-repair on staging node", "node", node.Name) + metav1.SetMetaDataAnnotation(&modifiedNode.ObjectMeta, karpenterDoNotRepairAnnotationKey, "true") + } + if node.Annotations[karpenterDoNotDisruptAnnotationKey] != "true" { + log.Info("Setting karpenter.sh/do-not-disrupt on staging node", "node", node.Name) + metav1.SetMetaDataAnnotation(&modifiedNode.ObjectMeta, karpenterDoNotDisruptAnnotationKey, "true") + } + if !annotationsEqual(node.Annotations, modifiedNode.Annotations) { + if err := r.Patch(ctx, modifiedNode, client.StrategicMergeFrom(node)); err != nil { + return fmt.Errorf("patching node for staging protection: %w", err) + } + } + + return nil +} + +// annotationsEqual reports whether two annotation maps are identical. +// It treats a nil map and an empty map as equal, unlike maps.Equal which +// returns false when one argument is nil and the other is a non-nil empty map. +// Kubernetes can return either (a freshly created object with no annotations +// has nil; one that had all annotations deleted may have an empty map), so +// this distinction matters for correct annotation comparison. +func annotationsEqual(a, b map[string]string) bool { + if len(a) != len(b) { + return false + } + for k, v := range a { + if b[k] != v { + return false + } + } + return true +} diff --git a/internal/controller/karpenter_test.go b/internal/controller/karpenter_test.go new file mode 100644 index 00000000..40ac9492 --- /dev/null +++ b/internal/controller/karpenter_test.go @@ -0,0 +1,1406 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +// karpenter_test.go contains unit tests for the Karpenter integration logic in +// karpenter.go and the Karpenter-specific paths in rollout.go and +// bootcnodepool_controller.go. Tests that require a live API server (envtest) +// live in rollout_karpenter_envtest_test.go. + +import ( + "context" + "testing" + + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + bootcv1alpha1 "github.com/bootc-dev/bootc-operator/api/v1alpha1" + testutil "github.com/bootc-dev/bootc-operator/test/util" +) + +// patchCountingClient wraps a client.Client and counts Patch calls to the +// Node object (by GVK). Used to assert that freeRebootSlot doesn't +// double-patch the Node. +type patchCountingClient struct { + client.Client + nodePatches int +} + +func (c *patchCountingClient) Patch( + ctx context.Context, + obj client.Object, + patch client.Patch, + opts ...client.PatchOption, +) error { + if _, ok := obj.(*corev1.Node); ok { + c.nodePatches++ + } + return c.Client.Patch(ctx, obj, patch, opts...) +} + +// newFakeReconciler builds a BootcNodePoolReconciler backed by a fake +// client pre-seeded with the given objects. It also returns the counting +// wrapper so callers can assert the number of Node patches. +func newFakeReconciler(objs ...client.Object) (*BootcNodePoolReconciler, *patchCountingClient) { + sc := runtime.NewScheme() + _ = scheme.AddToScheme(sc) + _ = bootcv1alpha1.AddToScheme(sc) + + base := fake.NewClientBuilder(). + WithScheme(sc). + WithObjects(objs...). + Build() + + counting := &patchCountingClient{Client: base} + r := &BootcNodePoolReconciler{Client: counting} + return r, counting +} + +func TestKarpenterRebootActive(t *testing.T) { + claim := &unstructured.Unstructured{Object: map[string]interface{}{ + "apiVersion": karpenterNodeClaimAPIVersion, + "kind": karpenterNodeClaimKind, + "metadata": map[string]interface{}{ + "name": "claim", + }, + "status": map[string]interface{}{ + "conditions": []interface{}{ + map[string]interface{}{"type": "Rebooting", "status": "True"}, + }, + }, + }} + node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{ + Name: "node", + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: karpenterNodeClaimAPIVersion, + Kind: karpenterNodeClaimKind, + Name: "claim", + UID: types.UID("claim-uid"), + }}, + }} + + r, _ := newFakeReconciler(node, claim) + active, _, _, err := r.isKarpenterDisruptionActive(context.Background(), node) + NewWithT(t).Expect(err).NotTo(HaveOccurred()) + NewWithT(t).Expect(active).To(BeTrue()) + + node.Spec.Taints = []corev1.Taint{{ + Key: karpenterRebootingTaintKey, + Effect: karpenterRebootingTaintEffect, + }} + claim.Object["status"] = map[string]interface{}{} + active, _, _, err = r.isKarpenterDisruptionActive(context.Background(), node) + NewWithT(t).Expect(err).NotTo(HaveOccurred()) + NewWithT(t).Expect(active).To(BeTrue()) +} + +// TestRestoreNodeState verifies that restoreNodeState removes owned Karpenter +// annotations and restores cordon state in a single patch, covering all +// combinations of ownership and cordon state. +func TestRestoreNodeState(t *testing.T) { + tests := []struct { + name string + ownsRepair bool + ownsDisrupt bool + wasCordoned string // "true", "false", or "" (absent) + nodeCordoned bool + nodeAnnotations map[string]string + + wantNodePatches int + wantCordoned bool + wantRepairPresent bool + wantDisruptPresent bool + }{ + { + name: "owns both annotations, operator cordoned: single patch removes all", + ownsRepair: true, + ownsDisrupt: true, + wasCordoned: "false", + nodeCordoned: true, + nodeAnnotations: map[string]string{ + karpenterDoNotRepairAnnotationKey: "true", + karpenterDoNotDisruptAnnotationKey: "true", + }, + wantNodePatches: 1, + wantCordoned: false, + wantRepairPresent: false, + wantDisruptPresent: false, + }, + { + name: "owns both annotations, pre-cordoned: single patch, cordon preserved", + ownsRepair: true, + ownsDisrupt: true, + wasCordoned: "true", + nodeCordoned: true, + nodeAnnotations: map[string]string{ + karpenterDoNotRepairAnnotationKey: "true", + karpenterDoNotDisruptAnnotationKey: "true", + }, + wantNodePatches: 1, + wantCordoned: true, + wantRepairPresent: false, + wantDisruptPresent: false, + }, + { + name: "owns only do-not-disrupt: removes only that one", + ownsRepair: false, + ownsDisrupt: true, + wasCordoned: "true", + nodeCordoned: true, + nodeAnnotations: map[string]string{ + karpenterDoNotRepairAnnotationKey: "true", + karpenterDoNotDisruptAnnotationKey: "true", + }, + wantNodePatches: 1, + wantCordoned: true, + wantRepairPresent: true, + wantDisruptPresent: false, + }, + { + name: "no karpenter ownership, operator cordoned: single uncordon patch", + ownsRepair: false, + ownsDisrupt: false, + wasCordoned: "false", + nodeCordoned: true, + wantNodePatches: 1, + wantCordoned: false, + }, + { + name: "no karpenter ownership, pre-cordoned: no patch needed", + ownsRepair: false, + ownsDisrupt: false, + wasCordoned: "true", + nodeCordoned: true, + wantNodePatches: 0, + wantCordoned: true, + }, + { + name: "nothing to do: zero patches", + ownsRepair: false, + ownsDisrupt: false, + wasCordoned: "", + nodeCordoned: false, + wantNodePatches: 0, + wantCordoned: false, + }, + { + // Karpenter annotations present on the node but set by an external + // actor, not the operator. The operator must not touch them. + name: "annotations present but not owned: zero patches, annotations preserved", + ownsRepair: false, + ownsDisrupt: false, + wasCordoned: "false", + nodeCordoned: true, + nodeAnnotations: map[string]string{ + karpenterDoNotRepairAnnotationKey: "true", + karpenterDoNotDisruptAnnotationKey: "true", + }, + wantNodePatches: 1, // uncordon only + wantCordoned: false, + wantRepairPresent: true, + wantDisruptPresent: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + ctx := context.Background() + + bnAnnotations := map[string]string{} + if tc.wasCordoned != "" { + bnAnnotations[bootcv1alpha1.AnnotationWasCordoned] = tc.wasCordoned + } + if tc.ownsRepair { + bnAnnotations[bootcv1alpha1.AnnotationOwnsKarpenterDoNotRepair] = "true" + } + if tc.ownsDisrupt { + bnAnnotations[bootcv1alpha1.AnnotationOwnsKarpenterDoNotDisrupt] = "true" + } + bn := testutil.NewNode("test-node", testImageDigestRefA) + bn.Annotations = bnAnnotations + + k8sNode := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + Annotations: tc.nodeAnnotations, + }, + Spec: corev1.NodeSpec{Unschedulable: tc.nodeCordoned}, + } + + r, counting := newFakeReconciler(bn, k8sNode) + + err := r.restoreNodeState(ctx, bn, k8sNode) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(counting.nodePatches).To(Equal(tc.wantNodePatches), "node patch count") + + var finalNode corev1.Node + g.Expect(counting.Client.Get(ctx, client.ObjectKey{Name: "test-node"}, &finalNode)). + To(Succeed()) + g.Expect(finalNode.Spec.Unschedulable).To(Equal(tc.wantCordoned), "cordon state") + if tc.wantRepairPresent { + g.Expect(finalNode.Annotations).To(HaveKey(karpenterDoNotRepairAnnotationKey)) + } else { + g.Expect(finalNode.Annotations).NotTo(HaveKey(karpenterDoNotRepairAnnotationKey)) + } + if tc.wantDisruptPresent { + g.Expect(finalNode.Annotations).To(HaveKey(karpenterDoNotDisruptAnnotationKey)) + } else { + g.Expect(finalNode.Annotations).NotTo(HaveKey(karpenterDoNotDisruptAnnotationKey)) + } + }) + } +} + +// TestAssignRebootSlotNodePatchCount verifies that assignRebootSlot issues at +// most one patch to the corev1.Node regardless of Karpenter ownership and +// prior cordon state. +func TestAssignRebootSlotNodePatchCount(t *testing.T) { + karpenterOwnerRef := metav1.OwnerReference{ + APIVersion: karpenterNodeClaimAPIVersion, + Kind: karpenterNodeClaimKind, + Name: "test-claim", + UID: "test-claim-uid", + } + + tests := []struct { + name string + karpenterOwned bool + preExistAnnotations map[string]string // annotations already on the Node + preCordoned bool + + wantNodePatches int + wantCordoned bool + wantRepairPresent bool + wantDisruptPresent bool + }{ + { + name: "karpenter owned, not cordoned: one patch for cordon + both annotations", + karpenterOwned: true, + wantNodePatches: 1, + wantCordoned: true, + wantRepairPresent: true, + wantDisruptPresent: true, + }, + { + name: "karpenter owned, pre-cordoned: one patch for annotations only", + karpenterOwned: true, + preCordoned: true, + wantNodePatches: 1, + wantCordoned: true, + wantRepairPresent: true, + wantDisruptPresent: true, + }, + { + name: "karpenter owned, both annotations pre-existing: zero node patches", + karpenterOwned: true, + preCordoned: true, + preExistAnnotations: map[string]string{ + karpenterDoNotRepairAnnotationKey: "true", + karpenterDoNotDisruptAnnotationKey: "true", + }, + wantNodePatches: 0, + wantCordoned: true, + wantRepairPresent: true, + wantDisruptPresent: true, + }, + { + name: "non-karpenter, not cordoned: one patch for cordon only", + karpenterOwned: false, + wantNodePatches: 1, + wantCordoned: true, + }, + { + name: "non-karpenter, pre-cordoned: zero node patches", + karpenterOwned: false, + preCordoned: true, + wantNodePatches: 0, + wantCordoned: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + ctx := context.Background() + + bn := testutil.NewNode("test-node", testImageDigestRefA) + + k8sNode := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + Annotations: tc.preExistAnnotations, + }, + Spec: corev1.NodeSpec{Unschedulable: tc.preCordoned}, + } + if tc.karpenterOwned { + k8sNode.OwnerReferences = []metav1.OwnerReference{karpenterOwnerRef} + } + + r, counting := newFakeReconciler(bn, k8sNode) + + err := r.assignRebootSlot(ctx, bn, k8sNode, nil) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(counting.nodePatches).To(Equal(tc.wantNodePatches), "node patch count") + + var finalNode corev1.Node + g.Expect(counting.Client.Get(ctx, client.ObjectKey{Name: "test-node"}, &finalNode)). + To(Succeed()) + g.Expect(finalNode.Spec.Unschedulable).To(Equal(tc.wantCordoned), "cordon state") + if tc.wantRepairPresent { + g.Expect(finalNode.Annotations).To(HaveKey(karpenterDoNotRepairAnnotationKey)) + } else { + g.Expect(finalNode.Annotations).NotTo(HaveKey(karpenterDoNotRepairAnnotationKey)) + } + if tc.wantDisruptPresent { + g.Expect(finalNode.Annotations).To(HaveKey(karpenterDoNotDisruptAnnotationKey)) + } else { + g.Expect(finalNode.Annotations).NotTo(HaveKey(karpenterDoNotDisruptAnnotationKey)) + } + + // The BootcNode must always have the slot annotation. + var finalBN bootcv1alpha1.BootcNode + g.Expect(counting.Client.Get(ctx, client.ObjectKey{Name: "test-node"}, &finalBN)). + To(Succeed()) + g.Expect(finalBN.Annotations).To(HaveKey(bootcv1alpha1.AnnotationInRebootSlot)) + }) + } +} + +// TestKarpenterNodeClaimAnnotationLifecycle verifies that assignRebootSlot +// applies do-not-repair and do-not-disrupt to the NodeClaim when absent, and +// that freeRebootSlot removes only annotations the operator owns. +func TestKarpenterNodeClaimAnnotationLifecycle(t *testing.T) { + type annotationPair struct { + claimKey string + ownershipAnnotation string + } + annotations := []annotationPair{ + { + karpenterDoNotDisruptAnnotationKey, + bootcv1alpha1.AnnotationOwnsKarpenterNodeClaimDoNotDisrupt, + }, + { + karpenterDoNotRepairAnnotationKey, + bootcv1alpha1.AnnotationOwnsKarpenterNodeClaimDoNotRepair, + }, + } + + tests := []struct { + name string + preExistingAnnotation bool + wantDuring bool + wantOwned bool + wantAfter bool + }{ + { + name: "operator-owned annotation is applied and removed", + wantDuring: true, + wantOwned: true, + wantAfter: false, + }, + { + name: "pre-existing annotation is preserved", + preExistingAnnotation: true, + wantDuring: true, + wantOwned: false, + wantAfter: true, + }, + } + + for _, ann := range annotations { + for _, tc := range tests { + t.Run(ann.claimKey+"/"+tc.name, func(t *testing.T) { + g := NewWithT(t) + ctx := context.Background() + + claim := &unstructured.Unstructured{Object: map[string]interface{}{ + "apiVersion": karpenterNodeClaimAPIVersion, + "kind": karpenterNodeClaimKind, + "metadata": map[string]interface{}{ + "name": "test-claim", + }, + }} + if tc.preExistingAnnotation { + claim.SetAnnotations(map[string]string{ann.claimKey: "true"}) + } + + node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: karpenterNodeClaimAPIVersion, + Kind: karpenterNodeClaimKind, + Name: "test-claim", + UID: types.UID("test-claim-uid"), + }}, + }} + bn := testutil.NewNode("test-node", testImageDigestRefA) + + r, _ := newFakeReconciler(bn, node, claim) + g.Expect(r.assignRebootSlot(ctx, bn, node, nil)).To(Succeed()) + + var duringClaim unstructured.Unstructured + duringClaim.SetAPIVersion(karpenterNodeClaimAPIVersion) + duringClaim.SetKind(karpenterNodeClaimKind) + g.Expect(r.Get(ctx, client.ObjectKey{Name: "test-claim"}, &duringClaim)). + To(Succeed()) + if tc.wantDuring { + g.Expect(duringClaim.GetAnnotations()). + To(HaveKeyWithValue(ann.claimKey, "true")) + } else { + g.Expect(duringClaim.GetAnnotations()).NotTo(HaveKey(ann.claimKey)) + } + + var duringBN bootcv1alpha1.BootcNode + g.Expect(r.Get(ctx, client.ObjectKey{Name: "test-node"}, &duringBN)).To(Succeed()) + if tc.wantOwned { + g.Expect(duringBN.Annotations).To(HaveKey(ann.ownershipAnnotation)) + } else { + g.Expect(duringBN.Annotations).NotTo(HaveKey(ann.ownershipAnnotation)) + } + + g.Expect(r.freeRebootSlot(ctx, &duringBN, node)).To(Succeed()) + + var afterClaim unstructured.Unstructured + afterClaim.SetAPIVersion(karpenterNodeClaimAPIVersion) + afterClaim.SetKind(karpenterNodeClaimKind) + g.Expect(r.Get(ctx, client.ObjectKey{Name: "test-claim"}, &afterClaim)). + To(Succeed()) + if tc.wantAfter { + g.Expect(afterClaim.GetAnnotations()).To(HaveKeyWithValue(ann.claimKey, "true")) + } else { + g.Expect(afterClaim.GetAnnotations()).NotTo(HaveKey(ann.claimKey)) + } + }) + } + } +} + +// TestFreeRebootSlotNodeClaimGone verifies that freeRebootSlot succeeds cleanly +// when the NodeClaim no longer exists by the time the slot is freed. +// +// The scenario unfolds as follows: +// +// 1. The operator assigns the node a reboot slot and, because the node is +// Karpenter-managed, sets do-not-repair / do-not-disrupt annotations on +// both the corev1.Node and its backing NodeClaim to prevent Karpenter from +// interfering while the OS update is in progress. It records ownership of +// those annotations via AnnotationOwnsKarpenterNodeClaimDoNotRepair and +// AnnotationOwnsKarpenterNodeClaimDoNotDisrupt on the BootcNode. +// +// 2. The node reboots and comes back running the new image. The reconciler +// calls freeRebootSlot → restoreNodeState, which fetches the NodeClaim via +// getKarpenterNodeClaim so it can remove the annotations it added. +// +// 3. Between step 1 and step 2 Karpenter forcefully replaced the node via a +// mechanism that the annotations do not block — for example a spot-instance +// interruption, node expiration (expireAfter), or manual NodeClaim +// deletion. (do-not-repair and do-not-disrupt only suppress Node Auto +// Repair and voluntary disruption respectively; they cannot prevent these +// forceful termination paths.) Karpenter deletes the NodeClaim and +// provisions a new one. The original NodeClaim therefore returns a 404 +// (IsNotFound) from the API server. +// +// getKarpenterNodeClaim treats a 404 (or no-match) as a nil return, so +// restoreNodeState skips the NodeClaim patch and returns nil. This test +// confirms the operator does not surface that 404 as an error, and that it +// still clears all slot/ownership annotations and uncordons the node, leaving +// state clean for the next reconcile cycle. +func TestFreeRebootSlotNodeClaimGone(t *testing.T) { + g := NewWithT(t) + ctx := context.Background() + + // Build a BootcNode with both NodeClaim ownership annotations set. + bn := testutil.NewNode("test-node", testImageDigestRefA) + bn.Annotations = map[string]string{ + bootcv1alpha1.AnnotationInRebootSlot: "", + bootcv1alpha1.AnnotationWasCordoned: "false", + bootcv1alpha1.AnnotationOwnsKarpenterNodeClaimDoNotRepair: "true", + bootcv1alpha1.AnnotationOwnsKarpenterNodeClaimDoNotDisrupt: "true", + } + + // Node has an owner reference pointing to a NodeClaim that no longer exists. + k8sNode := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: karpenterNodeClaimAPIVersion, + Kind: karpenterNodeClaimKind, + Name: "gone-claim", + UID: types.UID("gone-uid"), + }}, + }, + Spec: corev1.NodeSpec{Unschedulable: true}, + } + + // The NodeClaim is intentionally absent from the fake client. + r, _ := newFakeReconciler(bn, k8sNode) + g.Expect(r.freeRebootSlot(ctx, bn, k8sNode)).To(Succeed()) + + // Cordon should be restored (was-cordoned=false). + var finalNode corev1.Node + g.Expect(r.Get(ctx, client.ObjectKey{Name: "test-node"}, &finalNode)).To(Succeed()) + g.Expect(finalNode.Spec.Unschedulable).To(BeFalse(), "node should be uncordoned") + + // All slot annotations should be cleared. + var finalBN bootcv1alpha1.BootcNode + g.Expect(r.Get(ctx, client.ObjectKey{Name: "test-node"}, &finalBN)).To(Succeed()) + g.Expect(finalBN.Annotations).NotTo(HaveKey(bootcv1alpha1.AnnotationInRebootSlot)) + g.Expect(finalBN.Annotations). + NotTo(HaveKey(bootcv1alpha1.AnnotationOwnsKarpenterNodeClaimDoNotRepair)) + g.Expect(finalBN.Annotations). + NotTo(HaveKey(bootcv1alpha1.AnnotationOwnsKarpenterNodeClaimDoNotDisrupt)) +} + +// TestDriveRolloutDefersWhenKarpenterRebootActive verifies that driveRollout +// skips slot assignment and drain for a node whose NodeClaim has +// Rebooting=True, leaving the BootcNode without an in-reboot-slot annotation. +func TestDriveRolloutDefersWhenKarpenterRebootActive(t *testing.T) { + g := NewWithT(t) + ctx := context.Background() + + // A staged BootcNode ready for a reboot slot. + bn := testutil.NewNode("staged-node", testImageDigestRefA, + testutil.WithBootedDigest(testDigestB), + testutil.WithNodeCondition( + bootcv1alpha1.NodeIdle, + metav1.ConditionFalse, + bootcv1alpha1.NodeReasonStaged, + ), + ) + + // The corresponding K8s Node is owned by a NodeClaim that is currently rebooting. + k8sNode := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "staged-node", + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: karpenterNodeClaimAPIVersion, + Kind: karpenterNodeClaimKind, + Name: "active-claim", + UID: types.UID("active-claim-uid"), + }}, + }, + } + + // NodeClaim with Rebooting=True. + claim := &unstructured.Unstructured{Object: map[string]interface{}{ + "apiVersion": karpenterNodeClaimAPIVersion, + "kind": karpenterNodeClaimKind, + "metadata": map[string]interface{}{ + "name": "active-claim", + }, + "status": map[string]interface{}{ + "nodeName": "staged-node", + "conditions": []interface{}{ + map[string]interface{}{"type": "Rebooting", "status": "True"}, + }, + }, + }} + + pool := testutil.NewPool("test-pool", testImageDigestRefA, testutil.WithWorkerSelector()) + pool.Status.TargetDigest = testDigestA + + r, _ := newFakeReconciler(bn, k8sNode, claim) + ownedNodes := map[string]*bootcv1alpha1.BootcNode{"staged-node": bn} + + rs, err := r.driveRollout(ctx, pool, ownedNodes) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(rs).NotTo(BeNil()) + + // The node must not have been assigned a reboot slot. + var finalBN bootcv1alpha1.BootcNode + g.Expect(r.Get(ctx, client.ObjectKey{Name: "staged-node"}, &finalBN)).To(Succeed()) + g.Expect(finalBN.Annotations).NotTo(HaveKey(bootcv1alpha1.AnnotationInRebootSlot), + "slot must not be assigned while Karpenter reboot is active") +} + +// TestFreeRebootSlotNodePatchCount verifies that freeRebootSlot issues at most +// one patch to the corev1.Node regardless of which combination of Karpenter +// annotations and cordon state are in play. +func TestFreeRebootSlotNodePatchCount(t *testing.T) { + tests := []struct { + name string + + // BootcNode annotation state + ownsRepair bool + ownsDisrupt bool + wasCordoned string // "true", "false", or "" (absent) + + // Node state + nodeCordoned bool + nodeAnnotations map[string]string + + // Expected outcomes + wantNodePatches int + wantCordoned bool + wantRepairPresent bool + wantDisruptPresent bool + }{ + { + // Karpenter annotations owned + operator cordoned: remove both + // annotations and uncordon in a single patch. + name: "karpenter owned, operator cordoned: one patch", + ownsRepair: true, + ownsDisrupt: true, + wasCordoned: "false", + nodeCordoned: true, + nodeAnnotations: map[string]string{ + karpenterDoNotRepairAnnotationKey: "true", + karpenterDoNotDisruptAnnotationKey: "true", + }, + wantNodePatches: 1, + wantCordoned: false, + wantRepairPresent: false, + wantDisruptPresent: false, + }, + { + // Karpenter annotations owned + node was pre-cordoned: remove + // annotations but leave the cordon, still a single patch. + name: "karpenter owned, pre-cordoned: one patch, cordon preserved", + ownsRepair: true, + ownsDisrupt: true, + wasCordoned: "true", + nodeCordoned: true, + nodeAnnotations: map[string]string{ + karpenterDoNotRepairAnnotationKey: "true", + karpenterDoNotDisruptAnnotationKey: "true", + }, + wantNodePatches: 1, + wantCordoned: true, + wantRepairPresent: false, + wantDisruptPresent: false, + }, + { + // No Karpenter ownership + operator cordoned: single uncordon patch. + name: "no karpenter, operator cordoned: one patch", + ownsRepair: false, + ownsDisrupt: false, + wasCordoned: "false", + nodeCordoned: true, + wantNodePatches: 1, + wantCordoned: false, + }, + { + // No Karpenter ownership + node was pre-cordoned: no node patch needed. + name: "no karpenter, pre-cordoned: zero patches", + ownsRepair: false, + ownsDisrupt: false, + wasCordoned: "true", + nodeCordoned: true, + wantNodePatches: 0, + wantCordoned: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + ctx := context.Background() + + // Build the BootcNode with the appropriate slot annotations. + bnAnnotations := map[string]string{ + bootcv1alpha1.AnnotationInRebootSlot: "", + } + if tc.wasCordoned != "" { + bnAnnotations[bootcv1alpha1.AnnotationWasCordoned] = tc.wasCordoned + } + if tc.ownsRepair { + bnAnnotations[bootcv1alpha1.AnnotationOwnsKarpenterDoNotRepair] = "true" + } + if tc.ownsDisrupt { + bnAnnotations[bootcv1alpha1.AnnotationOwnsKarpenterDoNotDisrupt] = "true" + } + bn := testutil.NewNode("test-node", testImageDigestRefA) + bn.Annotations = bnAnnotations + + // Build the corev1.Node. + k8sNode := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + Annotations: tc.nodeAnnotations, + }, + Spec: corev1.NodeSpec{Unschedulable: tc.nodeCordoned}, + } + + r, counting := newFakeReconciler(bn, k8sNode) + + err := r.freeRebootSlot(ctx, bn, k8sNode) + g.Expect(err).NotTo(HaveOccurred()) + + g.Expect(counting.nodePatches).To(Equal(tc.wantNodePatches), + "unexpected number of Node patches") + + // Verify the final node state from the fake store. + var finalNode corev1.Node + g.Expect(counting.Client.Get(ctx, client.ObjectKey{Name: "test-node"}, &finalNode)). + To(Succeed()) + g.Expect(finalNode.Spec.Unschedulable).To(Equal(tc.wantCordoned), "cordon state") + if tc.ownsRepair { + g.Expect(finalNode.Annotations).NotTo(HaveKey(karpenterDoNotRepairAnnotationKey)) + } + if tc.ownsDisrupt { + g.Expect(finalNode.Annotations).NotTo(HaveKey(karpenterDoNotDisruptAnnotationKey)) + } + + // Verify slot annotations are cleared from the BootcNode. + var finalBN bootcv1alpha1.BootcNode + g.Expect(counting.Client.Get(ctx, client.ObjectKey{Name: "test-node"}, &finalBN)). + To(Succeed()) + g.Expect(finalBN.Annotations).NotTo(HaveKey(bootcv1alpha1.AnnotationInRebootSlot)) + g.Expect(finalBN.Annotations).NotTo(HaveKey(bootcv1alpha1.AnnotationWasCordoned)) + g.Expect(finalBN.Annotations). + NotTo(HaveKey(bootcv1alpha1.AnnotationOwnsKarpenterDoNotRepair)) + g.Expect(finalBN.Annotations). + NotTo(HaveKey(bootcv1alpha1.AnnotationOwnsKarpenterDoNotDisrupt)) + }) + } +} + +// TestIsKarpenterDisruptionActive verifies that isKarpenterDisruptionActive detects +// all relevant disruption signals: +// - Node taint karpenter.sh/rebooting:NoSchedule +// - Node taint karpenter.sh/disrupted:NoSchedule +// - Node DeletionTimestamp set +// - NodeClaim DeletionTimestamp set +// - NodeClaim condition DisruptionReason=True +// - NodeClaim condition InstanceTerminating=True +// - NodeClaim missing but owner ref present → claimMissing=true +func TestIsKarpenterDisruptionActive(t *testing.T) { + now := metav1.Now() + + tests := []struct { + name string + // node setup + nodeTaints []corev1.Taint + nodeDeletionTS *metav1.Time + nodeKarpenterOwned bool + // claim setup + claimConditions []interface{} + claimDeletionTS *metav1.Time + claimAbsent bool // NodeClaim owner ref present but object 404 + // expected + wantActive bool + wantClaimMissing bool + }{ + { + name: "idle node, no owner ref: inactive", + wantActive: false, + }, + { + name: "rebooting taint: active", + nodeKarpenterOwned: true, + nodeTaints: []corev1.Taint{{ + Key: karpenterRebootingTaintKey, + Effect: karpenterRebootingTaintEffect, + }}, + wantActive: true, + }, + { + name: "disrupted taint: active", + nodeKarpenterOwned: true, + nodeTaints: []corev1.Taint{{ + Key: karpenterDisruptedTaintKey, + Effect: karpenterRebootingTaintEffect, + }}, + wantActive: true, + }, + { + name: "node deletion timestamp: active", + nodeKarpenterOwned: true, + nodeDeletionTS: &now, + wantActive: true, + }, + { + name: "NodeClaim deletion timestamp: active", + nodeKarpenterOwned: true, + claimDeletionTS: &now, + wantActive: true, + }, + { + name: "NodeClaim Rebooting=True condition: active", + nodeKarpenterOwned: true, + claimConditions: []interface{}{ + map[string]interface{}{"type": "Rebooting", "status": "True"}, + }, + wantActive: true, + }, + { + name: "NodeClaim DisruptionReason=True condition: active", + nodeKarpenterOwned: true, + claimConditions: []interface{}{ + map[string]interface{}{"type": "DisruptionReason", "status": "True"}, + }, + wantActive: true, + }, + { + name: "NodeClaim InstanceTerminating=True condition: active", + nodeKarpenterOwned: true, + claimConditions: []interface{}{ + map[string]interface{}{"type": "InstanceTerminating", "status": "True"}, + }, + wantActive: true, + }, + { + name: "NodeClaim condition status False: not active", + nodeKarpenterOwned: true, + claimConditions: []interface{}{ + map[string]interface{}{"type": "DisruptionReason", "status": "False"}, + }, + wantActive: false, + }, + { + name: "NodeClaim absent but owner ref present: claimMissing=true", + nodeKarpenterOwned: true, + claimAbsent: true, + wantActive: true, + wantClaimMissing: true, + }, + { + name: "idle Karpenter node, no conditions: not active", + nodeKarpenterOwned: true, + wantActive: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + ctx := context.Background() + + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + // Kubernetes requires a finalizer for DeletionTimestamp to be set. + Finalizers: func() []string { + if tc.nodeDeletionTS != nil { + return []string{"test/finalizer"} + } + return nil + }(), + DeletionTimestamp: tc.nodeDeletionTS, + }, + Spec: corev1.NodeSpec{Taints: tc.nodeTaints}, + } + + var objs []client.Object + objs = append(objs, node) + + if tc.nodeKarpenterOwned && !tc.claimAbsent { + claimObj := map[string]interface{}{ + "apiVersion": karpenterNodeClaimAPIVersion, + "kind": karpenterNodeClaimKind, + "metadata": map[string]interface{}{ + "name": "test-claim", + }, + } + if len(tc.claimConditions) > 0 { + claimObj["status"] = map[string]interface{}{ + "conditions": tc.claimConditions, + } + } + claim := &unstructured.Unstructured{Object: claimObj} + if tc.claimDeletionTS != nil { + claim.SetDeletionTimestamp(tc.claimDeletionTS) + claim.SetFinalizers([]string{"test/finalizer"}) + } + objs = append(objs, claim) + } + + if tc.nodeKarpenterOwned { + node.OwnerReferences = []metav1.OwnerReference{{ + APIVersion: karpenterNodeClaimAPIVersion, + Kind: karpenterNodeClaimKind, + Name: "test-claim", + UID: types.UID("test-claim-uid"), + }} + } + + r, _ := newFakeReconciler(objs...) + active, claimMissing, _, err := r.isKarpenterDisruptionActive(ctx, node) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(active).To(Equal(tc.wantActive), "active") + g.Expect(claimMissing).To(Equal(tc.wantClaimMissing), "claimMissing") + }) + } +} + +// TestAssignRebootSlotAnnotationValueCheck verifies that assignRebootSlot +// takes ownership of and overwrites a Karpenter annotation that already exists +// but has a value other than "true" (e.g. "false"). Karpenter only honors the +// literal value "true"; any other value leaves the node unprotected. +func TestAssignRebootSlotAnnotationValueCheck(t *testing.T) { + tests := []struct { + name string + existingValue string // value already set on the Node annotation + wantOwned bool // does the BootcNode get an ownership annotation? + }{ + { + name: "annotation absent: operator takes ownership", + wantOwned: true, + }, + { + name: "annotation already true: operator does not take ownership", + existingValue: "true", + wantOwned: false, + }, + { + name: "annotation is false: operator overwrites and takes ownership", + existingValue: "false", + wantOwned: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + ctx := context.Background() + + nodeAnnotations := map[string]string{} + if tc.existingValue != "" { + nodeAnnotations[karpenterDoNotDisruptAnnotationKey] = tc.existingValue + nodeAnnotations[karpenterDoNotRepairAnnotationKey] = tc.existingValue + } + + k8sNode := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + Annotations: nodeAnnotations, + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: karpenterNodeClaimAPIVersion, + Kind: karpenterNodeClaimKind, + Name: "test-claim", + UID: types.UID("test-claim-uid"), + }}, + }, + } + claim := &unstructured.Unstructured{Object: map[string]interface{}{ + "apiVersion": karpenterNodeClaimAPIVersion, + "kind": karpenterNodeClaimKind, + "metadata": map[string]interface{}{"name": "test-claim"}, + }} + bn := testutil.NewNode("test-node", testImageDigestRefA) + + r, _ := newFakeReconciler(bn, k8sNode, claim) + g.Expect(r.assignRebootSlot(ctx, bn, k8sNode, nil)).To(Succeed()) + + // The Node annotation must always be "true" after assignment. + var finalNode corev1.Node + g.Expect(r.Get(ctx, client.ObjectKey{Name: "test-node"}, &finalNode)).To(Succeed()) + g.Expect(finalNode.Annotations[karpenterDoNotDisruptAnnotationKey]).To(Equal("true")) + g.Expect(finalNode.Annotations[karpenterDoNotRepairAnnotationKey]).To(Equal("true")) + + // Ownership annotation on BootcNode. + var finalBN bootcv1alpha1.BootcNode + g.Expect(r.Get(ctx, client.ObjectKey{Name: "test-node"}, &finalBN)).To(Succeed()) + if tc.wantOwned { + g.Expect(finalBN.Annotations).To(HaveKey(bootcv1alpha1.AnnotationOwnsKarpenterDoNotDisrupt)) + g.Expect(finalBN.Annotations).To(HaveKey(bootcv1alpha1.AnnotationOwnsKarpenterDoNotRepair)) + } else { + g.Expect(finalBN.Annotations).NotTo(HaveKey(bootcv1alpha1.AnnotationOwnsKarpenterDoNotDisrupt)) + g.Expect(finalBN.Annotations).NotTo(HaveKey(bootcv1alpha1.AnnotationOwnsKarpenterDoNotRepair)) + } + }) + } +} + +// TestEnsureKarpenterStagingProtection verifies that ensureKarpenterStagingProtection +// applies do-not-repair and do-not-disrupt to a Karpenter-owned node in the +// Staging state, before any reboot slot is assigned. +func TestEnsureKarpenterStagingProtection(t *testing.T) { + g := NewWithT(t) + ctx := context.Background() + + bn := testutil.NewNode("staging-node", testImageDigestRefA, + testutil.WithBootedDigest(testDigestB), + testutil.WithNodeCondition(bootcv1alpha1.NodeIdle, metav1.ConditionFalse, bootcv1alpha1.NodeReasonStaging), + ) + + k8sNode := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "staging-node", + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: karpenterNodeClaimAPIVersion, + Kind: karpenterNodeClaimKind, + Name: "staging-claim", + UID: types.UID("staging-claim-uid"), + }}, + }, + } + claim := &unstructured.Unstructured{Object: map[string]interface{}{ + "apiVersion": karpenterNodeClaimAPIVersion, + "kind": karpenterNodeClaimKind, + "metadata": map[string]interface{}{"name": "staging-claim"}, + }} + + r, _ := newFakeReconciler(bn, k8sNode, claim) + g.Expect(r.ensureKarpenterStagingProtection(ctx, bn, k8sNode)).To(Succeed()) + + // Node must have both Karpenter protection annotations. + var finalNode corev1.Node + g.Expect(r.Get(ctx, client.ObjectKey{Name: "staging-node"}, &finalNode)).To(Succeed()) + g.Expect(finalNode.Annotations[karpenterDoNotRepairAnnotationKey]).To(Equal("true")) + g.Expect(finalNode.Annotations[karpenterDoNotDisruptAnnotationKey]).To(Equal("true")) + + // NodeClaim must have both annotations. + var finalClaim unstructured.Unstructured + finalClaim.SetAPIVersion(karpenterNodeClaimAPIVersion) + finalClaim.SetKind(karpenterNodeClaimKind) + g.Expect(r.Get(ctx, client.ObjectKey{Name: "staging-claim"}, &finalClaim)).To(Succeed()) + g.Expect(finalClaim.GetAnnotations()[karpenterDoNotRepairAnnotationKey]).To(Equal("true")) + g.Expect(finalClaim.GetAnnotations()[karpenterDoNotDisruptAnnotationKey]).To(Equal("true")) + + // BootcNode must record ownership. + var finalBN bootcv1alpha1.BootcNode + g.Expect(r.Get(ctx, client.ObjectKey{Name: "staging-node"}, &finalBN)).To(Succeed()) + g.Expect(finalBN.Annotations).To(HaveKey(bootcv1alpha1.AnnotationOwnsKarpenterDoNotRepair)) + g.Expect(finalBN.Annotations).To(HaveKey(bootcv1alpha1.AnnotationOwnsKarpenterDoNotDisrupt)) + g.Expect(finalBN.Annotations).To(HaveKey(bootcv1alpha1.AnnotationOwnsKarpenterNodeClaimDoNotRepair)) + g.Expect(finalBN.Annotations).To(HaveKey(bootcv1alpha1.AnnotationOwnsKarpenterNodeClaimDoNotDisrupt)) + + // No reboot slot should have been assigned. + g.Expect(finalBN.Annotations).NotTo(HaveKey(bootcv1alpha1.AnnotationInRebootSlot)) +} + +// TestEnsureKarpenterStagingProtectionNonKarpenter verifies that +// ensureKarpenterStagingProtection is a no-op for non-Karpenter nodes. +func TestEnsureKarpenterStagingProtectionNonKarpenter(t *testing.T) { + g := NewWithT(t) + ctx := context.Background() + + bn := testutil.NewNode("staging-node", testImageDigestRefA, + testutil.WithBootedDigest(testDigestB), + testutil.WithNodeCondition(bootcv1alpha1.NodeIdle, metav1.ConditionFalse, bootcv1alpha1.NodeReasonStaging), + ) + // No Karpenter owner reference. + k8sNode := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "staging-node"}} + + r, counting := newFakeReconciler(bn, k8sNode) + g.Expect(r.ensureKarpenterStagingProtection(ctx, bn, k8sNode)).To(Succeed()) + + // No patches to the Node should have been issued. + g.Expect(counting.nodePatches).To(Equal(0)) + + // BootcNode must be unchanged. + var finalBN bootcv1alpha1.BootcNode + g.Expect(r.Get(ctx, client.ObjectKey{Name: "staging-node"}, &finalBN)).To(Succeed()) + g.Expect(finalBN.Annotations).NotTo(HaveKey(bootcv1alpha1.AnnotationOwnsKarpenterDoNotRepair)) +} + +// TestKarpenterStagingProtectionToSlotFreeLifecycle verifies the full +// staging-protection → slot-assignment → slot-free lifecycle for a +// Karpenter-owned node. Specifically it checks that: +// +// 1. Ownership annotations written by ensureKarpenterStagingProtection are not +// lost when assignRebootSlot subsequently runs (idempotency). +// 2. freeRebootSlot removes the Karpenter annotations from both the Node and +// the NodeClaim, and clears all ownership annotations from the BootcNode. +func TestKarpenterStagingProtectionToSlotFreeLifecycle(t *testing.T) { + g := NewWithT(t) + ctx := context.Background() + + bn := testutil.NewNode("karp-node", testImageDigestRefA, + testutil.WithBootedDigest(testDigestB), + testutil.WithNodeCondition(bootcv1alpha1.NodeIdle, metav1.ConditionFalse, bootcv1alpha1.NodeReasonStaging), + ) + k8sNode := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "karp-node", + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: karpenterNodeClaimAPIVersion, + Kind: karpenterNodeClaimKind, + Name: "karp-claim", + UID: types.UID("karp-claim-uid"), + }}, + }, + } + claim := &unstructured.Unstructured{Object: map[string]interface{}{ + "apiVersion": karpenterNodeClaimAPIVersion, + "kind": karpenterNodeClaimKind, + "metadata": map[string]interface{}{"name": "karp-claim"}, + }} + + r, _ := newFakeReconciler(bn, k8sNode, claim) + + // Step 1: staging protection is applied (simulates image pull in progress). + g.Expect(r.ensureKarpenterStagingProtection(ctx, bn, k8sNode)).To(Succeed()) + + // The BootcNode should carry all four ownership annotations. + var afterStaging bootcv1alpha1.BootcNode + g.Expect(r.Get(ctx, client.ObjectKey{Name: "karp-node"}, &afterStaging)).To(Succeed()) + g.Expect(afterStaging.Annotations).To(HaveKey(bootcv1alpha1.AnnotationOwnsKarpenterDoNotRepair)) + g.Expect(afterStaging.Annotations).To(HaveKey(bootcv1alpha1.AnnotationOwnsKarpenterDoNotDisrupt)) + g.Expect(afterStaging.Annotations).To(HaveKey(bootcv1alpha1.AnnotationOwnsKarpenterNodeClaimDoNotRepair)) + g.Expect(afterStaging.Annotations).To(HaveKey(bootcv1alpha1.AnnotationOwnsKarpenterNodeClaimDoNotDisrupt)) + + // Step 2: slot assignment runs (simulates image staged and slot available). + // Re-fetch the node to get the current annotation state from the fake store. + var currentNode corev1.Node + g.Expect(r.Get(ctx, client.ObjectKey{Name: "karp-node"}, ¤tNode)).To(Succeed()) + g.Expect(r.assignRebootSlot(ctx, &afterStaging, ¤tNode, nil)).To(Succeed()) + + var afterSlot bootcv1alpha1.BootcNode + g.Expect(r.Get(ctx, client.ObjectKey{Name: "karp-node"}, &afterSlot)).To(Succeed()) + // All four ownership annotations must still be present — assignRebootSlot + // must not clear or duplicate them. + g.Expect(afterSlot.Annotations).To(HaveKey(bootcv1alpha1.AnnotationInRebootSlot)) + g.Expect(afterSlot.Annotations).To(HaveKey(bootcv1alpha1.AnnotationOwnsKarpenterDoNotRepair)) + g.Expect(afterSlot.Annotations).To(HaveKey(bootcv1alpha1.AnnotationOwnsKarpenterDoNotDisrupt)) + g.Expect(afterSlot.Annotations).To(HaveKey(bootcv1alpha1.AnnotationOwnsKarpenterNodeClaimDoNotRepair)) + g.Expect(afterSlot.Annotations).To(HaveKey(bootcv1alpha1.AnnotationOwnsKarpenterNodeClaimDoNotDisrupt)) + + // Step 3: slot is freed (simulates successful reboot and Ready node). + g.Expect(r.Get(ctx, client.ObjectKey{Name: "karp-node"}, ¤tNode)).To(Succeed()) + g.Expect(r.freeRebootSlot(ctx, &afterSlot, ¤tNode)).To(Succeed()) + + // All slot and ownership annotations must be gone from the BootcNode. + var afterFree bootcv1alpha1.BootcNode + g.Expect(r.Get(ctx, client.ObjectKey{Name: "karp-node"}, &afterFree)).To(Succeed()) + g.Expect(afterFree.Annotations).NotTo(HaveKey(bootcv1alpha1.AnnotationInRebootSlot)) + g.Expect(afterFree.Annotations).NotTo(HaveKey(bootcv1alpha1.AnnotationOwnsKarpenterDoNotRepair)) + g.Expect(afterFree.Annotations).NotTo(HaveKey(bootcv1alpha1.AnnotationOwnsKarpenterDoNotDisrupt)) + g.Expect(afterFree.Annotations).NotTo(HaveKey(bootcv1alpha1.AnnotationOwnsKarpenterNodeClaimDoNotRepair)) + g.Expect(afterFree.Annotations).NotTo(HaveKey(bootcv1alpha1.AnnotationOwnsKarpenterNodeClaimDoNotDisrupt)) + + // Karpenter annotations must be removed from the Node. + var finalNode corev1.Node + g.Expect(r.Get(ctx, client.ObjectKey{Name: "karp-node"}, &finalNode)).To(Succeed()) + g.Expect(finalNode.Annotations).NotTo(HaveKey(karpenterDoNotRepairAnnotationKey)) + g.Expect(finalNode.Annotations).NotTo(HaveKey(karpenterDoNotDisruptAnnotationKey)) + + // Karpenter annotations must be removed from the NodeClaim. + var finalClaim unstructured.Unstructured + finalClaim.SetAPIVersion(karpenterNodeClaimAPIVersion) + finalClaim.SetKind(karpenterNodeClaimKind) + g.Expect(r.Get(ctx, client.ObjectKey{Name: "karp-claim"}, &finalClaim)).To(Succeed()) + g.Expect(finalClaim.GetAnnotations()).NotTo(HaveKey(karpenterDoNotRepairAnnotationKey)) + g.Expect(finalClaim.GetAnnotations()).NotTo(HaveKey(karpenterDoNotDisruptAnnotationKey)) +} + +// TestDriveRolloutStagingProtection verifies that driveRollout applies Karpenter +// protection to staging nodes before they receive a reboot slot. +func TestDriveRolloutStagingProtection(t *testing.T) { + g := NewWithT(t) + ctx := context.Background() + + // A staging BootcNode (image pull in progress, not yet staged). + bn := testutil.NewNode("staging-node", testImageDigestRefA, + testutil.WithBootedDigest(testDigestB), + testutil.WithNodeCondition( + bootcv1alpha1.NodeIdle, + metav1.ConditionFalse, + bootcv1alpha1.NodeReasonStaging, + ), + ) + + k8sNode := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "staging-node", + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: karpenterNodeClaimAPIVersion, + Kind: karpenterNodeClaimKind, + Name: "staging-claim", + UID: types.UID("staging-claim-uid"), + }}, + }, + } + claim := &unstructured.Unstructured{Object: map[string]interface{}{ + "apiVersion": karpenterNodeClaimAPIVersion, + "kind": karpenterNodeClaimKind, + "metadata": map[string]interface{}{"name": "staging-claim"}, + "status": map[string]interface{}{"nodeName": "staging-node"}, + }} + + pool := testutil.NewPool("test-pool", testImageDigestRefA, testutil.WithWorkerSelector()) + pool.Status.TargetDigest = testDigestA + + r, _ := newFakeReconciler(bn, k8sNode, claim) + ownedNodes := map[string]*bootcv1alpha1.BootcNode{"staging-node": bn} + + _, err := r.driveRollout(ctx, pool, ownedNodes) + g.Expect(err).NotTo(HaveOccurred()) + + // Node must have Karpenter protection applied. + var finalNode corev1.Node + g.Expect(r.Get(ctx, client.ObjectKey{Name: "staging-node"}, &finalNode)).To(Succeed()) + g.Expect(finalNode.Annotations[karpenterDoNotRepairAnnotationKey]).To(Equal("true"), + "do-not-repair must be set during staging") + g.Expect(finalNode.Annotations[karpenterDoNotDisruptAnnotationKey]).To(Equal("true"), + "do-not-disrupt must be set during staging") + + // The node must not have been assigned a reboot slot (it's still staging). + var finalBN bootcv1alpha1.BootcNode + g.Expect(r.Get(ctx, client.ObjectKey{Name: "staging-node"}, &finalBN)).To(Succeed()) + g.Expect(finalBN.Annotations).NotTo(HaveKey(bootcv1alpha1.AnnotationInRebootSlot), + "slot must not be assigned during staging") +} + +// TestDriveRolloutDefersWhenKarpenterDisruptionActive verifies that driveRollout +// defers slot assignment when Karpenter has committed a disruption (not just a +// reboot). +func TestDriveRolloutDefersWhenKarpenterDisruptionActive(t *testing.T) { + g := NewWithT(t) + ctx := context.Background() + + // A staged BootcNode ready for a reboot slot. + bn := testutil.NewNode("staged-node", testImageDigestRefA, + testutil.WithBootedDigest(testDigestB), + testutil.WithNodeCondition( + bootcv1alpha1.NodeIdle, + metav1.ConditionFalse, + bootcv1alpha1.NodeReasonStaged, + ), + ) + + // The Node has the disrupted taint (Karpenter voluntary disruption committed). + k8sNode := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "staged-node", + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: karpenterNodeClaimAPIVersion, + Kind: karpenterNodeClaimKind, + Name: "active-claim", + UID: types.UID("active-claim-uid"), + }}, + }, + Spec: corev1.NodeSpec{ + Taints: []corev1.Taint{{ + Key: karpenterDisruptedTaintKey, + Effect: karpenterRebootingTaintEffect, + }}, + }, + } + claim := &unstructured.Unstructured{Object: map[string]interface{}{ + "apiVersion": karpenterNodeClaimAPIVersion, + "kind": karpenterNodeClaimKind, + "metadata": map[string]interface{}{"name": "active-claim"}, + "status": map[string]interface{}{"nodeName": "staged-node"}, + }} + + pool := testutil.NewPool("test-pool", testImageDigestRefA, testutil.WithWorkerSelector()) + pool.Status.TargetDigest = testDigestA + + r, _ := newFakeReconciler(bn, k8sNode, claim) + ownedNodes := map[string]*bootcv1alpha1.BootcNode{"staged-node": bn} + + rs, err := r.driveRollout(ctx, pool, ownedNodes) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(rs).NotTo(BeNil()) + + // The node must not have been assigned a reboot slot. + var finalBN bootcv1alpha1.BootcNode + g.Expect(r.Get(ctx, client.ObjectKey{Name: "staged-node"}, &finalBN)).To(Succeed()) + g.Expect(finalBN.Annotations).NotTo(HaveKey(bootcv1alpha1.AnnotationInRebootSlot), + "slot must not be assigned while Karpenter disruption is active") +} + +// TestDriveRolloutDefersWhenNodeClaimAbsent verifies that driveRollout defers +// slot assignment when the Karpenter NodeClaim owner reference is present but +// the NodeClaim object is missing (404). +func TestDriveRolloutDefersWhenNodeClaimAbsent(t *testing.T) { + g := NewWithT(t) + ctx := context.Background() + + bn := testutil.NewNode("staged-node", testImageDigestRefA, + testutil.WithBootedDigest(testDigestB), + testutil.WithNodeCondition( + bootcv1alpha1.NodeIdle, + metav1.ConditionFalse, + bootcv1alpha1.NodeReasonStaged, + ), + ) + + // The Node has a Karpenter owner reference, but the NodeClaim does not exist. + k8sNode := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "staged-node", + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: karpenterNodeClaimAPIVersion, + Kind: karpenterNodeClaimKind, + Name: "missing-claim", + UID: types.UID("missing-claim-uid"), + }}, + }, + } + // The NodeClaim is intentionally absent from the fake client. + + pool := testutil.NewPool("test-pool", testImageDigestRefA, testutil.WithWorkerSelector()) + pool.Status.TargetDigest = testDigestA + + r, _ := newFakeReconciler(bn, k8sNode) + ownedNodes := map[string]*bootcv1alpha1.BootcNode{"staged-node": bn} + + rs, err := r.driveRollout(ctx, pool, ownedNodes) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(rs).NotTo(BeNil()) + + // Slot must not have been assigned — the absent NodeClaim is unsafe. + var finalBN bootcv1alpha1.BootcNode + g.Expect(r.Get(ctx, client.ObjectKey{Name: "staged-node"}, &finalBN)).To(Succeed()) + g.Expect(finalBN.Annotations).NotTo(HaveKey(bootcv1alpha1.AnnotationInRebootSlot), + "slot must not be assigned when NodeClaim owner is missing") + + // A requeue should be scheduled to retry. + g.Expect(rs.requeueAfter).To(BeNumerically(">", 0), + "requeueAfter must be set to retry after missing NodeClaim") +} + +// TestDriveRolloutKarpenterRequeueFallback verifies that driveRollout sets a +// periodic requeue when Karpenter-owned nodes are active (staging or staged), +// to ensure NodeClaim status changes are eventually observed even if the +// NodeClaim watch is not registered. +func TestDriveRolloutKarpenterRequeueFallback(t *testing.T) { + g := NewWithT(t) + ctx := context.Background() + + // A staged Karpenter-owned node. The BootcNode carries the ownership + // annotation that staging protection would have set in a prior reconcile — + // driveRollout uses these annotations to detect Karpenter-owned staged nodes + // without an extra API call per node. + bn := testutil.NewNode("staged-node", testImageDigestRefA, + testutil.WithBootedDigest(testDigestB), + testutil.WithNodeCondition( + bootcv1alpha1.NodeIdle, + metav1.ConditionFalse, + bootcv1alpha1.NodeReasonStaged, + ), + testutil.WithNodeAnnotation(bootcv1alpha1.AnnotationOwnsKarpenterDoNotRepair, "true"), + ) + k8sNode := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "staged-node", + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: karpenterNodeClaimAPIVersion, + Kind: karpenterNodeClaimKind, + Name: "test-claim", + UID: types.UID("test-claim-uid"), + }}, + }, + } + claim := &unstructured.Unstructured{Object: map[string]interface{}{ + "apiVersion": karpenterNodeClaimAPIVersion, + "kind": karpenterNodeClaimKind, + "metadata": map[string]interface{}{"name": "test-claim"}, + "status": map[string]interface{}{"nodeName": "staged-node"}, + }} + + pool := testutil.NewPool("test-pool", testImageDigestRefA, testutil.WithWorkerSelector()) + pool.Status.TargetDigest = testDigestA + + r, _ := newFakeReconciler(bn, k8sNode, claim) + ownedNodes := map[string]*bootcv1alpha1.BootcNode{"staged-node": bn} + + rs, err := r.driveRollout(ctx, pool, ownedNodes) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(rs).NotTo(BeNil()) + + // requeueAfter must be set to the fallback interval. + g.Expect(rs.requeueAfter).To(Equal(karpenterMissingClaimRequeue), + "requeueAfter must be set to the Karpenter fallback requeue interval") +} diff --git a/internal/controller/rollout.go b/internal/controller/rollout.go index f1c3d069..e1fc4992 100644 --- a/internal/controller/rollout.go +++ b/internal/controller/rollout.go @@ -5,6 +5,7 @@ package controller import ( "context" "fmt" + "maps" "slices" "strings" "time" @@ -14,6 +15,7 @@ import ( corev1 "k8s.io/api/core/v1" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/kubectl/pkg/drain" @@ -140,6 +142,51 @@ func (r *BootcNodePoolReconciler) driveRollout( return rs, nil } + // Apply Karpenter protection as soon as a node starts staging, before a + // reboot slot is even assigned. This closes the window where Karpenter + // could disrupt a node while its image pull is in progress. + // + // karpenterActive tracks whether any Karpenter-owned node is currently in + // the staging or staged state, so the periodic requeue fallback below can + // be set only when needed. + karpenterActive := false + for _, bn := range rs.staging { + var node corev1.Node + if err := r.Get(ctx, types.NamespacedName{Name: bn.Name}, &node); err != nil { + return nil, fmt.Errorf("fetching node %s: %w", bn.Name, err) + } + if isNodeOwnedByKarpenterNodeClaim(&node) { + karpenterActive = true + } + if err := r.ensureKarpenterStagingProtection(ctx, bn, &node); err != nil { + return nil, fmt.Errorf("applying Karpenter staging protection to %s: %w", bn.Name, err) + } + } + // Also check staged nodes: use the BootcNode Karpenter ownership + // annotations as a proxy, since staged nodes have already passed through + // ensureKarpenterStagingProtection (or assignRebootSlot) which records + // ownership. This avoids extra API calls for each staged node here. + if !karpenterActive { + for _, bn := range rs.staged { + if metav1.HasAnnotation(bn.ObjectMeta, bootcv1alpha1.AnnotationOwnsKarpenterDoNotRepair) || + metav1.HasAnnotation(bn.ObjectMeta, bootcv1alpha1.AnnotationOwnsKarpenterDoNotDisrupt) { + karpenterActive = true + break + } + } + } + + // Periodic requeue fallback: if any Karpenter-owned nodes are active + // (staging or staged), set a fallback requeue so that NodeClaim status + // changes are eventually observed even when the NodeClaim watch is not + // registered (e.g. Karpenter installed after bootc-operator started). + if karpenterActive { + // Only set if nothing sooner is already scheduled. + if rs.requeueAfter == 0 || karpenterMissingClaimRequeue < rs.requeueAfter { + rs.requeueAfter = karpenterMissingClaimRequeue + } + } + maxUnavail, err := resolveMaxUnavailable(pool, rs.nodeCount()) if err != nil { return nil, err @@ -168,7 +215,26 @@ func (r *BootcNodePoolReconciler) driveRollout( if err := r.Get(ctx, types.NamespacedName{Name: bn.Name}, &node); err != nil { return nil, fmt.Errorf("fetching node %s: %w", bn.Name, err) } - if err := r.assignRebootSlot(ctx, bn, &node); err != nil { + // isKarpenterDisruptionActive fetches the NodeClaim as part of its + // disruption checks. When disruption is not active, it returns the + // claim so assignRebootSlot can reuse it without a second API call. + active, claimMissing, nodeClaim, err := r.isKarpenterDisruptionActive(ctx, &node) + if err != nil { + return nil, fmt.Errorf("checking Karpenter disruption for %s: %w", bn.Name, err) + } + if claimMissing { + log.Info("Deferring reboot slot: Karpenter NodeClaim is unexpectedly absent, will retry", + "node", bn.Name) + if rs.requeueAfter == 0 || karpenterMissingClaimRequeue < rs.requeueAfter { + rs.requeueAfter = karpenterMissingClaimRequeue + } + continue + } + if active { + log.Info("Deferring reboot slot: Karpenter disruption is active", "node", bn.Name) + continue + } + if err := r.assignRebootSlot(ctx, bn, &node, nodeClaim); err != nil { return nil, fmt.Errorf("assigning reboot slot to %s: %w", bn.Name, err) } } @@ -182,53 +248,237 @@ func (r *BootcNodePoolReconciler) driveRollout( if bn.Spec.DesiredImageState == bootcv1alpha1.DesiredImageStateBooted { continue } + var node corev1.Node + if err := r.Get(ctx, types.NamespacedName{Name: bn.Name}, &node); err != nil { + return nil, fmt.Errorf("fetching node %s: %w", bn.Name, err) + } + active, claimMissing, _, err := r.isKarpenterDisruptionActive(ctx, &node) + if err != nil { + return nil, fmt.Errorf("checking Karpenter disruption for %s: %w", bn.Name, err) + } + if claimMissing { + log.Info("Deferring drain: Karpenter NodeClaim is unexpectedly absent, will retry", + "node", bn.Name) + if rs.requeueAfter == 0 || karpenterMissingClaimRequeue < rs.requeueAfter { + rs.requeueAfter = karpenterMissingClaimRequeue + } + continue + } + if active { + log.Info("Deferring drain: Karpenter disruption is active", "node", bn.Name) + continue + } r.ensureDrain(ctx, pool, bn) } return rs, nil } -// assignRebootSlot marks a BootcNode as occupying a reboot slot and -// cordons the corresponding K8s Node. It sets the in-reboot-slot -// annotation on the BootcNode, records prior cordon state in the -// was-cordoned annotation, and cordons the node. All operations are -// idempotent. +// assignRebootSlot marks a BootcNode as occupying a reboot slot and cordons +// the corresponding K8s Node. It sets the in-reboot-slot annotation on the +// BootcNode and records the prior cordon state in the was-cordoned annotation. +// For nodes owned by a Karpenter NodeClaim, it also sets the +// karpenter.sh/do-not-repair and karpenter.sh/do-not-disrupt annotations on +// the K8s Node to prevent Karpenter from replacing or disrupting the node +// during the update. Ownership of those annotations is tracked on the BootcNode +// so that restoreNodeState can remove only the ones the controller added. +// +// Note: these annotations are best-effort coordination signals, not a +// distributed lock. Karpenter can commit a disruption or reboot between the +// isKarpenterDisruptionActive check and the start of the drain. +// +// All operations are idempotent. +// +// prefetchedClaim may be the NodeClaim already fetched by a preceding +// isKarpenterDisruptionActive call. When non-nil and the node is +// Karpenter-owned, it is used directly to avoid a redundant API round-trip. func (r *BootcNodePoolReconciler) assignRebootSlot( ctx context.Context, bn *bootcv1alpha1.BootcNode, node *corev1.Node, + prefetchedClaim *unstructured.Unstructured, ) error { log := logf.FromContext(ctx) - // Set annotations on the BootcNode if not already present. - if !metav1.HasAnnotation(bn.ObjectMeta, bootcv1alpha1.AnnotationInRebootSlot) { - log.Info("Assigning reboot slot", "node", bn.Name) - modifiedBN := bn.DeepCopy() - if modifiedBN.Annotations == nil { - modifiedBN.Annotations = map[string]string{} + karpenterOwned := isNodeOwnedByKarpenterNodeClaim(node) + + // Resolve the NodeClaim. Use the caller-supplied prefetched claim when + // available to avoid a second API round-trip. If no claim was pre-fetched + // (e.g. the node is not Karpenter-owned or the call site doesn't supply + // one), fall back to fetching it here. + // + // claim may be nil even when karpenterOwned is true: + // - The NodeClaim CRD is not installed (Karpenter is not present on this + // cluster, so the API server returns a no-match error). + // - The NodeClaim is absent (karpenterClaimAbsent) — callers must have + // already deferred via isKarpenterDisruptionActive before reaching here. + // In both cases we skip NodeClaim annotation management and proceed with + // only Node-level fencing. + var claim *unstructured.Unstructured + if karpenterOwned { + if prefetchedClaim != nil { + claim = prefetchedClaim + } else { + var claimResult karpenterNodeClaimResult + var err error + claim, claimResult, err = r.getKarpenterNodeClaim(ctx, node) + if err != nil { + return fmt.Errorf("getting NodeClaim: %w", err) + } + _ = claimResult // result is implicit: nil claim means absent/missing } - modifiedBN.Annotations[bootcv1alpha1.AnnotationInRebootSlot] = "" - // Record whether the node was already cordoned before us. + } + // Only manage NodeClaim annotations when the claim was successfully fetched. + // karpenterClaimAbsent should have been caught by isKarpenterDisruptionActive + // earlier, but skip NodeClaim patching defensively if we reach here anyway. + claimAvailable := claim != nil + + // Build the desired BootcNode annotation state and patch only if anything + // changed. We apply every annotation the slot requires unconditionally and + // let the comparison at the end decide whether an API call is needed. This + // avoids a separate upfront guard expression that would duplicate the same + // per-annotation conditions. + modifiedBN := bn.DeepCopy() + + // Mark the node as occupying a reboot slot. + metav1.SetMetaDataAnnotation(&modifiedBN.ObjectMeta, bootcv1alpha1.AnnotationInRebootSlot, "") + + // Record the prior cordon state so restoreNodeState can leave the node as + // it was found (i.e. not uncordon a node that was already cordoned). Only + // set this on first assignment — if the slot annotation is already present + // we must not overwrite the original recorded value. + if !metav1.HasAnnotation(bn.ObjectMeta, bootcv1alpha1.AnnotationInRebootSlot) { + wasCordoned := "false" + if node.Spec.Unschedulable { - modifiedBN.Annotations[bootcv1alpha1.AnnotationWasCordoned] = "true" + wasCordoned = "true" + } + + metav1.SetMetaDataAnnotation( + &modifiedBN.ObjectMeta, + bootcv1alpha1.AnnotationWasCordoned, + wasCordoned, + ) + } + + // For Karpenter-managed nodes, record ownership of the do-not-repair and + // do-not-disrupt annotations on the Node and NodeClaim. We take ownership + // when the annotation is absent or not already set to "true". Karpenter + // only honors the literal value "true"; any other value (e.g. "false") is + // ineffective, so we overwrite it and track ownership so we clean it up. + if karpenterOwned { + if node.Annotations[karpenterDoNotRepairAnnotationKey] != "true" { + metav1.SetMetaDataAnnotation( + &modifiedBN.ObjectMeta, + bootcv1alpha1.AnnotationOwnsKarpenterDoNotRepair, + "true", + ) + } + + if node.Annotations[karpenterDoNotDisruptAnnotationKey] != "true" { + metav1.SetMetaDataAnnotation( + &modifiedBN.ObjectMeta, + bootcv1alpha1.AnnotationOwnsKarpenterDoNotDisrupt, + "true", + ) + } + + if claimAvailable { + claimAnnotations := claim.GetAnnotations() + + if claimAnnotations[karpenterDoNotRepairAnnotationKey] != "true" { + metav1.SetMetaDataAnnotation( + &modifiedBN.ObjectMeta, + bootcv1alpha1.AnnotationOwnsKarpenterNodeClaimDoNotRepair, + "true", + ) + } + + if claimAnnotations[karpenterDoNotDisruptAnnotationKey] != "true" { + metav1.SetMetaDataAnnotation( + &modifiedBN.ObjectMeta, + bootcv1alpha1.AnnotationOwnsKarpenterNodeClaimDoNotDisrupt, + "true", + ) + } + } + } + + if !annotationsEqual(bn.Annotations, modifiedBN.Annotations) { + if !metav1.HasAnnotation(bn.ObjectMeta, bootcv1alpha1.AnnotationInRebootSlot) { + log.Info("Assigning reboot slot", "node", bn.Name) } else { - modifiedBN.Annotations[bootcv1alpha1.AnnotationWasCordoned] = "false" + log.Info("Updating reboot slot annotations", "node", bn.Name) } if err := r.Patch(ctx, modifiedBN, client.MergeFrom(bn)); err != nil { return fmt.Errorf("annotating BootcNode: %w", err) } - *bn = *modifiedBN } - // Cordon the K8s Node if not already cordoned. + // Apply the Karpenter fence to the NodeClaim after recording ownership on + // the BootcNode. This ordering prevents a successful NodeClaim patch from + // becoming orphaned if the durable BootcNode update fails. + // + // NodeClaim annotations are a supplemental signal, not a distributed lock. + // Karpenter can already have selected or queued a disruption by the time + // these annotations are written; the Node-level checks (taint, deletion + // timestamp, disruption conditions) are the authoritative guard. + if claimAvailable && karpenterOwned { + claimAnnotations := claim.GetAnnotations() + modifiedClaimAnnotations := map[string]string{} + maps.Copy(modifiedClaimAnnotations, claimAnnotations) + + if claimAnnotations[karpenterDoNotRepairAnnotationKey] != "true" { + modifiedClaimAnnotations[karpenterDoNotRepairAnnotationKey] = "true" + } + + if claimAnnotations[karpenterDoNotDisruptAnnotationKey] != "true" { + modifiedClaimAnnotations[karpenterDoNotDisruptAnnotationKey] = "true" + } + + if !annotationsEqual(claimAnnotations, modifiedClaimAnnotations) { + modifiedClaim := claim.DeepCopy() + modifiedClaim.SetAnnotations(modifiedClaimAnnotations) + + if err := r.Patch(ctx, modifiedClaim, client.MergeFrom(claim)); err != nil { + return fmt.Errorf("patching Karpenter NodeClaim: %w", err) + } + } + } + + // Cordon the K8s Node and set the Node-level Karpenter annotations. + // Skip the patch entirely when nothing needs to change. + modifiedNode := node.DeepCopy() if !node.Spec.Unschedulable { log.Info("Cordoning node", "node", node.Name) - modifiedNode := node.DeepCopy() modifiedNode.Spec.Unschedulable = true + } + + if karpenterOwned { + if node.Annotations[karpenterDoNotRepairAnnotationKey] != "true" { + log.Info("Setting karpenter.sh/do-not-repair on node", "node", node.Name) + metav1.SetMetaDataAnnotation( + &modifiedNode.ObjectMeta, + karpenterDoNotRepairAnnotationKey, + "true", + ) + } + + if node.Annotations[karpenterDoNotDisruptAnnotationKey] != "true" { + log.Info("Setting karpenter.sh/do-not-disrupt on node", "node", node.Name) + metav1.SetMetaDataAnnotation( + &modifiedNode.ObjectMeta, + karpenterDoNotDisruptAnnotationKey, + "true", + ) + } + } + + if modifiedNode.Spec.Unschedulable != node.Spec.Unschedulable || + !annotationsEqual(node.Annotations, modifiedNode.Annotations) { if err := r.Patch(ctx, modifiedNode, client.StrategicMergeFrom(node)); err != nil { - return fmt.Errorf("cordoning node: %w", err) + return fmt.Errorf("patching node: %w", err) } - *node = *modifiedNode } return nil @@ -272,12 +522,17 @@ func (r *BootcNodePoolReconciler) freeRebootSlot( bn *bootcv1alpha1.BootcNode, node *corev1.Node, ) error { - if err := r.restoreCordonState(ctx, bn, node); err != nil { + if err := r.restoreNodeState(ctx, bn, node); err != nil { return err } - // Remove both reboot slot annotations from the BootcNode. + // Remove all reboot slot annotations from the BootcNode, including the + // Karpenter ownership trackers if present. modified := bn.DeepCopy() + delete(modified.Annotations, bootcv1alpha1.AnnotationOwnsKarpenterDoNotRepair) + delete(modified.Annotations, bootcv1alpha1.AnnotationOwnsKarpenterDoNotDisrupt) + delete(modified.Annotations, bootcv1alpha1.AnnotationOwnsKarpenterNodeClaimDoNotRepair) + delete(modified.Annotations, bootcv1alpha1.AnnotationOwnsKarpenterNodeClaimDoNotDisrupt) delete(modified.Annotations, bootcv1alpha1.AnnotationWasCordoned) delete(modified.Annotations, bootcv1alpha1.AnnotationInRebootSlot) if err := r.Patch(ctx, modified, client.MergeFrom(bn)); err != nil { diff --git a/internal/controller/rollout_karpenter_envtest_test.go b/internal/controller/rollout_karpenter_envtest_test.go new file mode 100644 index 00000000..e3b3fb7b --- /dev/null +++ b/internal/controller/rollout_karpenter_envtest_test.go @@ -0,0 +1,256 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controller + +// TestKarpenterAnnotations is a table-driven integration test that covers +// Karpenter NodeClaim annotation management across all distinct scenarios. Each +// row exercises a different combination of node ownership, pre-existing +// annotations, and cordon state. + +import ( + "context" + "fmt" + "testing" + + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + + bootcv1alpha1 "github.com/bootc-dev/bootc-operator/api/v1alpha1" + testutil "github.com/bootc-dev/bootc-operator/test/util" +) + +func TestKarpenterAnnotations(t *testing.T) { + tests := []struct { + // name is the subtest name and is used to derive unique resource names. + name string + + // Node setup. + nodeClaim bool // whether to attach a NodeClaim owner reference + preExistingAnnotations map[string]string // annotations to set on the Node before the rollout + preCordon bool // whether the Node starts cordoned + + // Expected BootcNode ownership annotations while the reboot slot is held. + wantOwnsRepair bool // AnnotationOwnsKarpenterDoNotRepair should be present + wantOwnsDisrupt bool // AnnotationOwnsKarpenterDoNotDisrupt should be present + + // Expected Node annotations while the reboot slot is held. + wantRepairDuring bool // karpenter.sh/do-not-repair should be present + wantDisruptDuring bool // karpenter.sh/do-not-disrupt should be present + + // Expected Node annotations after the slot is freed. + wantRepairAfter bool // karpenter.sh/do-not-repair should still be present + wantDisruptAfter bool // karpenter.sh/do-not-disrupt should still be present + + // Expected Node cordon state after the slot is freed. + wantCordonedAfter bool + }{ + // A plain NodeClaim-owned node with no pre-existing Karpenter state. + // The operator adds both annotations on slot assignment, records + // ownership, and removes them when the slot is freed. + { + name: "nodeclaim owned, no pre-existing annotations", + nodeClaim: true, + wantOwnsRepair: true, + wantOwnsDisrupt: true, + wantRepairDuring: true, + wantDisruptDuring: true, + wantRepairAfter: false, + wantDisruptAfter: false, + wantCordonedAfter: false, + }, + // A node with both Karpenter annotations already set by an external + // actor and no NodeClaim owner reference. The operator must not claim + // ownership of either annotation and must leave both untouched + // throughout, including after the slot is freed. + { + name: "no nodeclaim, both annotations pre-existing", + nodeClaim: false, + preExistingAnnotations: map[string]string{ + karpenterDoNotRepairAnnotationKey: "true", + karpenterDoNotDisruptAnnotationKey: "true", + }, + wantOwnsRepair: false, + wantOwnsDisrupt: false, + wantRepairDuring: true, // preserved as-is + wantDisruptDuring: true, // preserved as-is + wantRepairAfter: true, // untouched by operator + wantDisruptAfter: true, // untouched by operator + wantCordonedAfter: false, + }, + // A NodeClaim-owned node that was already cordoned before the rollout. + // The operator adds both Karpenter annotations and removes them on slot + // free, but the pre-existing cordon must be preserved throughout. + { + name: "nodeclaim owned, pre-cordoned", + nodeClaim: true, + preCordon: true, + wantOwnsRepair: true, + wantOwnsDisrupt: true, + wantRepairDuring: true, + wantDisruptDuring: true, + wantRepairAfter: false, + wantDisruptAfter: false, + wantCordonedAfter: true, // pre-existing cordon preserved + }, + // A NodeClaim-owned node where only do-not-repair is pre-existing. + // The operator must not claim do-not-repair (already set externally) + // but must add, own, and later remove do-not-disrupt independently. + // This verifies that per-annotation ownership tracking is applied + // independently rather than as an all-or-nothing decision. + { + name: "nodeclaim owned, only do-not-repair pre-existing", + nodeClaim: true, + preExistingAnnotations: map[string]string{ + karpenterDoNotRepairAnnotationKey: "true", + }, + wantOwnsRepair: false, // pre-existing; operator does not claim it + wantOwnsDisrupt: true, // not pre-existing; operator adds and owns it + wantRepairDuring: true, // preserved from pre-existing + wantDisruptDuring: true, // added by operator + wantRepairAfter: true, // not owned; left in place + wantDisruptAfter: false, // owned; removed by operator + wantCordonedAfter: false, + }, + } + + for i, tc := range tests { + tc := tc + // Derive unique, collision-free resource names from the subtest index. + nodeName := fmt.Sprintf("karp-ann-%d-node", i) + poolName := fmt.Sprintf("karp-ann-%d-pool", i) + + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + g.SetDefaultEventuallyTimeout(pollTimeout) + g.SetDefaultEventuallyPollingInterval(pollInterval) + ctx := context.Background() + + // Build and create the K8s Node. + node := testutil.NewK8sNode(nodeName, testutil.WorkerLabels()) + if tc.nodeClaim { + node.OwnerReferences = []metav1.OwnerReference{{ + APIVersion: karpenterNodeClaimAPIVersion, + Kind: karpenterNodeClaimKind, + Name: nodeName + "-claim", + UID: types.UID(nodeName + "-claim-uid"), + }} + } + if len(tc.preExistingAnnotations) > 0 { + node.Annotations = tc.preExistingAnnotations + } + if tc.preCordon { + node.Spec.Unschedulable = true + } + g.Expect(k8sClient.Create(ctx, node)).To(Succeed()) + t.Cleanup(func() { _ = k8sClient.Delete(ctx, node) }) + + // Create pool and wait for BootcNode. + pool := testutil.NewPool(poolName, testImageDigestRefB, + testutil.WithWorkerSelector(), + testutil.WithMaxUnavailable(intstr.FromInt32(1)), + ) + g.Expect(k8sClient.Create(ctx, pool)).To(Succeed()) + t.Cleanup(func() { _ = k8sClient.Delete(ctx, pool) }) + + g.Eventually(func() error { + return k8sClient.Get( + ctx, + client.ObjectKey{Name: nodeName}, + &bootcv1alpha1.BootcNode{}, + ) + }).Should(Succeed()) + + simulateDaemonStatus(g, ctx, nodeName, testDigestA, bootcv1alpha1.NodeReasonStaged) + + // Wait for slot assignment and drain completion. + g.Eventually(func() (*bootcv1alpha1.BootcNode, error) { + var bn bootcv1alpha1.BootcNode + err := k8sClient.Get(ctx, client.ObjectKey{Name: nodeName}, &bn) + return &bn, err + }).Should(And( + HaveField("Annotations", HaveKey(bootcv1alpha1.AnnotationInRebootSlot)), + HaveField("Spec.DesiredImageState", Equal(bootcv1alpha1.DesiredImageStateBooted)), + ), "BootcNode should have reboot slot and desiredImageState Booted") + + // Assert ownership annotations on BootcNode. + g.Eventually(func() (map[string]string, error) { + var bn bootcv1alpha1.BootcNode + err := k8sClient.Get(ctx, client.ObjectKey{Name: nodeName}, &bn) + return bn.Annotations, err + }).Should(matchOwnership(tc.wantOwnsRepair, tc.wantOwnsDisrupt), + "BootcNode ownership annotations during slot") + + // Assert Karpenter annotations on Node. + g.Eventually(func() (map[string]string, error) { + var n corev1.Node + err := k8sClient.Get(ctx, client.ObjectKey{Name: nodeName}, &n) + return n.Annotations, err + }).Should(matchKarpenterAnnotations(tc.wantRepairDuring, tc.wantDisruptDuring), + "Node Karpenter annotations during slot") + + // Simulate successful reboot and node becoming Ready. + simulateDaemonStatus(g, ctx, nodeName, testDigestB, bootcv1alpha1.NodeReasonIdle) + setNodeReady(g, ctx, nodeName) + + // Wait for slot to be freed. + g.Eventually(func() (map[string]string, error) { + var bn bootcv1alpha1.BootcNode + err := k8sClient.Get(ctx, client.ObjectKey{Name: nodeName}, &bn) + return bn.Annotations, err + }).Should(And( + Not(HaveKey(bootcv1alpha1.AnnotationInRebootSlot)), + Not(HaveKey(bootcv1alpha1.AnnotationOwnsKarpenterDoNotRepair)), + Not(HaveKey(bootcv1alpha1.AnnotationOwnsKarpenterDoNotDisrupt)), + ), "BootcNode should have no slot or ownership annotations after slot free") + + // Assert final Node annotation state. + g.Eventually(func() (map[string]string, error) { + var n corev1.Node + err := k8sClient.Get(ctx, client.ObjectKey{Name: nodeName}, &n) + return n.Annotations, err + }).Should(matchKarpenterAnnotations(tc.wantRepairAfter, tc.wantDisruptAfter), + "Node Karpenter annotations after slot free") + + // Assert final cordon state. + g.Eventually(func() (bool, error) { + var n corev1.Node + err := k8sClient.Get(ctx, client.ObjectKey{Name: nodeName}, &n) + return n.Spec.Unschedulable, err + }).Should(Equal(tc.wantCordonedAfter), "Node cordon state after slot free") + }) + } +} + +// matchOwnership returns a Gomega matcher for the BootcNode annotation map +// that asserts the presence or absence of the two Karpenter ownership +// annotations independently. +func matchOwnership(wantRepair, wantDisrupt bool) OmegaMatcher { + repair := Not(HaveKey(bootcv1alpha1.AnnotationOwnsKarpenterDoNotRepair)) + if wantRepair { + repair = HaveKey(bootcv1alpha1.AnnotationOwnsKarpenterDoNotRepair) + } + disrupt := Not(HaveKey(bootcv1alpha1.AnnotationOwnsKarpenterDoNotDisrupt)) + if wantDisrupt { + disrupt = HaveKey(bootcv1alpha1.AnnotationOwnsKarpenterDoNotDisrupt) + } + return And(repair, disrupt) +} + +// matchKarpenterAnnotations returns a Gomega matcher for the Node annotation +// map that asserts the presence or absence of the two Karpenter annotations +// independently. +func matchKarpenterAnnotations(wantRepair, wantDisrupt bool) OmegaMatcher { + repair := Not(HaveKey(karpenterDoNotRepairAnnotationKey)) + if wantRepair { + repair = HaveKey(karpenterDoNotRepairAnnotationKey) + } + disrupt := Not(HaveKey(karpenterDoNotDisruptAnnotationKey)) + if wantDisrupt { + disrupt = HaveKey(karpenterDoNotDisruptAnnotationKey) + } + return And(repair, disrupt) +}