feat(api/placement): classify capacity-shortage reason for logs and traces

packages/api/internal/orchestrator/placement/errors.go

@@ -4,12 +4,50 @@ import (
 	"fmt"
 )
 
+// CapacityShortageReason classifies why placement found no node with room for
+// the request. It is for server-side logs and traces only; it never changes
+// the client-facing message.
+type CapacityShortageReason string
+
+const (
+	// ShortageUnknown is the zero value: the shortage was not classified.
+	ShortageUnknown CapacityShortageReason = ""
+	// ShortageHugepagePool: every sampled node that matched CPU, features and
+	// labels failed the hugepage memory admission (MemoryAdmission) — the
+	// memory pool, not CPU, is the binding constraint.
+	ShortageHugepagePool CapacityShortageReason = "hugepage_pool_full"
+	// ShortageNodeRefusals: nodes answered ResourceExhausted (running cap,
+	// starting cap, a guest still stopping, or not accepting) until the
+	// deadline, rather than failing admission up front.
+	ShortageNodeRefusals CapacityShortageReason = "node_refusals"
+	// ShortageNoCandidate: no node remained to try — every one excluded, or
+	// the cluster drained mid-placement.
+	ShortageNoCandidate CapacityShortageReason = "no_candidate_node"
+)
+
 // NoNodesAvailableError reports no capacity: no candidate node at all, or
-// nothing but ResourceExhausted refusals until the deadline.
-type NoNodesAvailableError struct{}
+// nothing but ResourceExhausted refusals until the deadline. The diagnostic
+// fields below say which, for logs and traces; they do not affect the
+// client-facing message, and the zero value keeps the original behaviour.
+type NoNodesAvailableError struct {
+	// Reason classifies the shortage (hugepage pool vs node refusals vs no
+	// candidate). ShortageUnknown when placement did not classify it.
+	Reason CapacityShortageReason
+	// Attempts is the completed create attempts before giving up.
+	Attempts int
+	// Refusals is the ResourceExhausted refusals seen across nodes.
+	Refusals int
+	// PoolsFull reports that at least one sampled node failed memory admission.
+	PoolsFull bool
+	// LastRefusal is the classified label of the last ResourceExhausted a node
+	// returned (see classifyRefusal); "" when no node refused.
+	LastRefusal string
+}
 
 var _ error = NoNodesAvailableError{}
 
+// Error keeps the original wording ("no nodes available") so callers matching
+// on it still work; the diagnostic fields carry the detail.
 func (NoNodesAvailableError) Error() string {
 	return "no nodes available"
 }

packages/api/internal/orchestrator/placement/memory_placement_test.go

@@ -264,3 +264,24 @@ func TestPlaceSandbox_ConcurrentPlacementsNeverOverfillANode(t *testing.T) {
 		})
 	}
 }
+
+// The capacity shortage carries a classified reason for logs and traces. Every
+// pool full is the hugepage-pool reason, not a bare unknown, so an operator can
+// tell a memory shortage from node refusals without reading the message text.
+func TestPlaceSandbox_EveryPoolFull_ReasonIsHugepagePool(t *testing.T) {
+	t.Parallel()
+
+	nodes := []*nodemanager.Node{
+		hugepageNode("a", 1600, 1600, 300, 2000),
+		hugepageNode("b", 1600, 1700, 200, 2000),
+	}
+
+	_, err := PlaceSandbox(t.Context(), NewBestOfK(DefaultBestOfKConfig()), nodes, nil,
+		hugepageSbxRequest(), Requirements{Memory: limit95}, false, nil)
+
+	var noNodes NoNodesAvailableError
+	require.ErrorAs(t, err, &noNodes)
+	assert.Equal(t, ShortageHugepagePool, noNodes.Reason)
+	assert.True(t, noNodes.PoolsFull)
+	assert.Equal(t, 0, noNodes.Attempts)
+}

packages/api/internal/orchestrator/placement/placement.go

@@ -2,10 +2,13 @@ package placement
 
 import (
 	"context"
+	"errors"
 	"maps"
+	"strings"
 	"time"
 
 	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
 	"go.uber.org/zap"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
@@ -172,15 +175,64 @@ func placeSandbox(
 
 	attempt := 0
 	refusals := 0
+	// lastRefusal is the classified label of the most recent node
+	// ResourceExhausted; "" until a node refuses.
+	lastRefusal := ""
 	// A node refused at the record step is excluded, so chooseNode no longer
 	// sees its full pool.
 	poolsFull := false
 
+	// noCapacity builds the capacity-shortage error with the diagnostics
+	// gathered so far, and records the classification on the span. The
+	// client-facing message is unchanged; this only enriches logs/traces.
+	noCapacity := func(reason CapacityShortageReason) NoNodesAvailableError {
+		e := NoNodesAvailableError{
+			Reason:      reason,
+			Attempts:    attempt,
+			Refusals:    refusals,
+			PoolsFull:   poolsFull,
+			LastRefusal: lastRefusal,
+		}
+		span.SetAttributes(
+			attribute.String("placement.capacity_shortage.reason", string(reason)),
+			attribute.Int("placement.capacity_shortage.attempts", attempt),
+			attribute.Int("placement.capacity_shortage.refusals", refusals),
+			attribute.Bool("placement.capacity_shortage.pools_full", poolsFull),
+			attribute.String("placement.capacity_shortage.last_refusal", lastRefusal),
+		)
+		logger.L().Warn(ctx, "Placement found no node with capacity",
+			logger.WithSandboxID(sbxRequest.GetSandbox().GetSandboxId()),
+			logger.WithTemplateID(sbxRequest.GetSandbox().GetTemplateId()),
+			zap.String("reason", string(reason)),
+			zap.Int("attempts", attempt),
+			zap.Int("refusals", refusals),
+			zap.Bool("pools_full", poolsFull),
+			zap.String("last_refusal", lastRefusal),
+			zap.Int64("requested_vcpu", sbxRequest.GetSandbox().GetVcpu()),
+			zap.Int64("requested_ram_mb", sbxRequest.GetSandbox().GetRamMb()),
+		)
+		return e
+	}
+
+	// classifyShortage picks the reason from what placement observed: a
+	// memory-admission failure outranks node refusals (the pool is the
+	// binding constraint), refusals outrank an empty candidate set.
+	classifyShortage := func() CapacityShortageReason {
+		switch {
+		case poolsFull:
+			return ShortageHugepagePool
+		case refusals > 0:
+			return ShortageNodeRefusals
+		default:
+			return ShortageNoCandidate
+		}
+	}
+
 	// answered reports what the nodes said to a caller that left while a
 	// detached create ran: never a timeout, so no node is pinned for a remap.
 	answered := func() (PlacementResult, error) {
 		if attempt == 0 {
-			return PlacementResult{}, NoNodesAvailableError{}
+			return PlacementResult{}, noCapacity(classifyShortage())
 		}
 
 		return PlacementResult{}, SandboxCreateError{Attempts: attempt, LastErr: lastCreateErr}
@@ -189,7 +241,7 @@ func placeSandbox(
 	// Nothing but capacity refusals before the deadline is capacity, not a slow placement.
 	deadline := func() (PlacementResult, error) {
 		if attempt == 0 && (refusals > 0 || poolsFull) {
-			return failed(NoNodesAvailableError{})
+			return failed(noCapacity(classifyShortage()))
 		}
 
 		return failed(PlacementTimeoutError{Attempts: attempt})
@@ -211,7 +263,7 @@ func placeSandbox(
 					return failed(SandboxCreateError{Attempts: attempt, LastErr: lastCreateErr})
 				}
 
-				return failed(NoNodesAvailableError{})
+				return failed(noCapacity(ShortageNoCandidate))
 			}
 
 			candidatesExcluded := nodesExcluded
@@ -232,8 +284,24 @@ func placeSandbox(
 					return failed(SandboxCreateError{Attempts: attempt, LastErr: lastCreateErr})
 				}
 
+				// A capacity shortage from chooseNode (full pools, or an
+				// unavailable node that would otherwise fit) is enriched with the
+				// loop's diagnostics; the chooseNode reason wins when the loop has
+				// not recorded its own pool-full yet. A mismatch (no compatible
+				// node) is returned as is.
+				var fromChoose NoNodesAvailableError
 				if poolsFull {
-					return failed(NoNodesAvailableError{})
+					return failed(noCapacity(ShortageHugepagePool))
+				}
+				if errors.As(err, &fromChoose) {
+					reason := fromChoose.Reason
+					if reason == ShortageUnknown {
+						reason = classifyShortage()
+					}
+					if fromChoose.PoolsFull {
+						poolsFull = true
+					}
+					return failed(noCapacity(reason))
 				}
 
 				return failed(err)
@@ -301,6 +369,9 @@ func placeSandbox(
 		switch statusCode {
 		case codes.ResourceExhausted:
 			refusals++
+			if ok {
+				lastRefusal = classifyRefusal(st)
+			}
 			failedNode.PlacementMetrics.Skip(sbxRequest.GetSandbox().GetSandboxId())
 			if ok && createRefusedByStatus(st) {
 				skipped[failedNode.ID] = struct{}{}
@@ -353,6 +424,33 @@ func placeSandbox(
 	return failed(SandboxCreateError{Attempts: attempt, LastErr: lastCreateErr})
 }
 
+// classifyRefusal labels a node's ResourceExhausted message so a capacity
+// shortage carries what the node was short of, not just "no nodes available".
+// It matches on the orchestrator's own wording; an unrecognised refusal is
+// labelled "other" rather than dropped, so a reworded node message degrades
+// to a coarse label instead of a blank one.
+func classifyRefusal(st *status.Status) string {
+	if st == nil {
+		return ""
+	}
+	msg := st.Message()
+	switch {
+	case strings.Contains(msg, "max number of running sandboxes"):
+		return "running_cap"
+	case strings.Contains(msg, "too many sandboxes starting"):
+		return "starting_cap"
+	case strings.Contains(msg, "not enough memory to map the guest"):
+		return "guest_memory_stopping"
+	case strings.Contains(msg, "not accepting new sandboxes"):
+		return "node_not_accepting"
+	case strings.Contains(msg, "being migrated"), strings.Contains(msg, "rebooting"),
+		strings.Contains(msg, "being checkpointed"), strings.Contains(msg, "persisting sandbox"):
+		return "lifecycle_busy"
+	default:
+		return "other"
+	}
+}
+
 // createRefusedForGood reports a create the node refused with the typed
 // detail that says no other node and no retry changes the answer.
 func createRefusedForGood(st *status.Status) bool {

packages/api/internal/orchestrator/placement/placement_best_of_K.go

@@ -167,8 +167,11 @@ func (b *BestOfK) chooseNode(_ context.Context, nodes []*nodemanager.Node, exclu
 	if bestNode == nil {
 		// Full pools, or an unavailable node that would fit, are a shortage that
 		// clears, not a mismatch.
-		if poolsFull || unavailableNodeWouldFit(nodes, excludedNodes, requirements, filterByLabels, requiredLabels) {
-			return nil, NoNodesAvailableError{}
+		if poolsFull {
+			return nil, NoNodesAvailableError{Reason: ShortageHugepagePool, PoolsFull: true}
+		}
+		if unavailableNodeWouldFit(nodes, excludedNodes, requirements, filterByLabels, requiredLabels) {
+			return nil, NoNodesAvailableError{Reason: ShortageNoCandidate}
 		}
 
 		return nil, FailedToPlaceSandboxError{