From 8de1f25afb3fc090c4759c05c9423e0bb7d5cfa9 Mon Sep 17 00:00:00 2001 From: adababys Date: Thu, 12 Nov 2026 22:11:17 +0800 Subject: [PATCH] feat(api/placement): classify capacity-shortage reason for logs and traces The "Failed to place sandbox: not enough capacity" error (503, capacity_unavailable) is returned whenever placement finds no node with room, but it never says which capacity was short: the hugepage memory pool (MemoryAdmission), node-side ResourceExhausted refusals (running cap, starting cap, a guest still stopping, node not accepting), or an empty candidate set. Operators cannot tell a memory shortage from a running-count cap without guessing. Add a CapacityShortageReason and diagnostic fields (Attempts, Refusals, PoolsFull, LastRefusal) to NoNodesAvailableError, populated at every return site and recorded on the place-sandbox span plus a structured warn log. classifyRefusal labels the node ResourceExhausted message so the last refusal reason is carried too. The client-facing message is unchanged (still "no nodes available" / the 503 text), and the zero value keeps the original behaviour, so no wire or API change. Diagnostics are server-side only. --- .../internal/orchestrator/placement/errors.go | 42 ++++++- .../placement/memory_placement_test.go | 21 ++++ .../orchestrator/placement/placement.go | 106 +++++++++++++++++- .../placement/placement_best_of_K.go | 7 +- 4 files changed, 168 insertions(+), 8 deletions(-) diff --git a/packages/api/internal/orchestrator/placement/errors.go b/packages/api/internal/orchestrator/placement/errors.go index 29d2081acd..0eb57a7c98 100644 --- a/packages/api/internal/orchestrator/placement/errors.go +++ b/packages/api/internal/orchestrator/placement/errors.go @@ -4,12 +4,50 @@ import ( "fmt" ) +// CapacityShortageReason classifies why placement found no node with room for +// the request. It is for server-side logs and traces only; it never changes +// the client-facing message. +type CapacityShortageReason string + +const ( + // ShortageUnknown is the zero value: the shortage was not classified. + ShortageUnknown CapacityShortageReason = "" + // ShortageHugepagePool: every sampled node that matched CPU, features and + // labels failed the hugepage memory admission (MemoryAdmission) — the + // memory pool, not CPU, is the binding constraint. + ShortageHugepagePool CapacityShortageReason = "hugepage_pool_full" + // ShortageNodeRefusals: nodes answered ResourceExhausted (running cap, + // starting cap, a guest still stopping, or not accepting) until the + // deadline, rather than failing admission up front. + ShortageNodeRefusals CapacityShortageReason = "node_refusals" + // ShortageNoCandidate: no node remained to try — every one excluded, or + // the cluster drained mid-placement. + ShortageNoCandidate CapacityShortageReason = "no_candidate_node" +) + // NoNodesAvailableError reports no capacity: no candidate node at all, or -// nothing but ResourceExhausted refusals until the deadline. -type NoNodesAvailableError struct{} +// nothing but ResourceExhausted refusals until the deadline. The diagnostic +// fields below say which, for logs and traces; they do not affect the +// client-facing message, and the zero value keeps the original behaviour. +type NoNodesAvailableError struct { + // Reason classifies the shortage (hugepage pool vs node refusals vs no + // candidate). ShortageUnknown when placement did not classify it. + Reason CapacityShortageReason + // Attempts is the completed create attempts before giving up. + Attempts int + // Refusals is the ResourceExhausted refusals seen across nodes. + Refusals int + // PoolsFull reports that at least one sampled node failed memory admission. + PoolsFull bool + // LastRefusal is the classified label of the last ResourceExhausted a node + // returned (see classifyRefusal); "" when no node refused. + LastRefusal string +} var _ error = NoNodesAvailableError{} +// Error keeps the original wording ("no nodes available") so callers matching +// on it still work; the diagnostic fields carry the detail. func (NoNodesAvailableError) Error() string { return "no nodes available" } diff --git a/packages/api/internal/orchestrator/placement/memory_placement_test.go b/packages/api/internal/orchestrator/placement/memory_placement_test.go index a6609d1457..c8fa423264 100644 --- a/packages/api/internal/orchestrator/placement/memory_placement_test.go +++ b/packages/api/internal/orchestrator/placement/memory_placement_test.go @@ -264,3 +264,24 @@ func TestPlaceSandbox_ConcurrentPlacementsNeverOverfillANode(t *testing.T) { }) } } + +// The capacity shortage carries a classified reason for logs and traces. Every +// pool full is the hugepage-pool reason, not a bare unknown, so an operator can +// tell a memory shortage from node refusals without reading the message text. +func TestPlaceSandbox_EveryPoolFull_ReasonIsHugepagePool(t *testing.T) { + t.Parallel() + + nodes := []*nodemanager.Node{ + hugepageNode("a", 1600, 1600, 300, 2000), + hugepageNode("b", 1600, 1700, 200, 2000), + } + + _, err := PlaceSandbox(t.Context(), NewBestOfK(DefaultBestOfKConfig()), nodes, nil, + hugepageSbxRequest(), Requirements{Memory: limit95}, false, nil) + + var noNodes NoNodesAvailableError + require.ErrorAs(t, err, &noNodes) + assert.Equal(t, ShortageHugepagePool, noNodes.Reason) + assert.True(t, noNodes.PoolsFull) + assert.Equal(t, 0, noNodes.Attempts) +} diff --git a/packages/api/internal/orchestrator/placement/placement.go b/packages/api/internal/orchestrator/placement/placement.go index 43e0de9514..bd2d594d62 100644 --- a/packages/api/internal/orchestrator/placement/placement.go +++ b/packages/api/internal/orchestrator/placement/placement.go @@ -2,10 +2,13 @@ package placement import ( "context" + "errors" "maps" + "strings" "time" "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" "go.uber.org/zap" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -172,15 +175,64 @@ func placeSandbox( attempt := 0 refusals := 0 + // lastRefusal is the classified label of the most recent node + // ResourceExhausted; "" until a node refuses. + lastRefusal := "" // A node refused at the record step is excluded, so chooseNode no longer // sees its full pool. poolsFull := false + // noCapacity builds the capacity-shortage error with the diagnostics + // gathered so far, and records the classification on the span. The + // client-facing message is unchanged; this only enriches logs/traces. + noCapacity := func(reason CapacityShortageReason) NoNodesAvailableError { + e := NoNodesAvailableError{ + Reason: reason, + Attempts: attempt, + Refusals: refusals, + PoolsFull: poolsFull, + LastRefusal: lastRefusal, + } + span.SetAttributes( + attribute.String("placement.capacity_shortage.reason", string(reason)), + attribute.Int("placement.capacity_shortage.attempts", attempt), + attribute.Int("placement.capacity_shortage.refusals", refusals), + attribute.Bool("placement.capacity_shortage.pools_full", poolsFull), + attribute.String("placement.capacity_shortage.last_refusal", lastRefusal), + ) + logger.L().Warn(ctx, "Placement found no node with capacity", + logger.WithSandboxID(sbxRequest.GetSandbox().GetSandboxId()), + logger.WithTemplateID(sbxRequest.GetSandbox().GetTemplateId()), + zap.String("reason", string(reason)), + zap.Int("attempts", attempt), + zap.Int("refusals", refusals), + zap.Bool("pools_full", poolsFull), + zap.String("last_refusal", lastRefusal), + zap.Int64("requested_vcpu", sbxRequest.GetSandbox().GetVcpu()), + zap.Int64("requested_ram_mb", sbxRequest.GetSandbox().GetRamMb()), + ) + return e + } + + // classifyShortage picks the reason from what placement observed: a + // memory-admission failure outranks node refusals (the pool is the + // binding constraint), refusals outrank an empty candidate set. + classifyShortage := func() CapacityShortageReason { + switch { + case poolsFull: + return ShortageHugepagePool + case refusals > 0: + return ShortageNodeRefusals + default: + return ShortageNoCandidate + } + } + // answered reports what the nodes said to a caller that left while a // detached create ran: never a timeout, so no node is pinned for a remap. answered := func() (PlacementResult, error) { if attempt == 0 { - return PlacementResult{}, NoNodesAvailableError{} + return PlacementResult{}, noCapacity(classifyShortage()) } return PlacementResult{}, SandboxCreateError{Attempts: attempt, LastErr: lastCreateErr} @@ -189,7 +241,7 @@ func placeSandbox( // Nothing but capacity refusals before the deadline is capacity, not a slow placement. deadline := func() (PlacementResult, error) { if attempt == 0 && (refusals > 0 || poolsFull) { - return failed(NoNodesAvailableError{}) + return failed(noCapacity(classifyShortage())) } return failed(PlacementTimeoutError{Attempts: attempt}) @@ -211,7 +263,7 @@ func placeSandbox( return failed(SandboxCreateError{Attempts: attempt, LastErr: lastCreateErr}) } - return failed(NoNodesAvailableError{}) + return failed(noCapacity(ShortageNoCandidate)) } candidatesExcluded := nodesExcluded @@ -232,8 +284,24 @@ func placeSandbox( return failed(SandboxCreateError{Attempts: attempt, LastErr: lastCreateErr}) } + // A capacity shortage from chooseNode (full pools, or an + // unavailable node that would otherwise fit) is enriched with the + // loop's diagnostics; the chooseNode reason wins when the loop has + // not recorded its own pool-full yet. A mismatch (no compatible + // node) is returned as is. + var fromChoose NoNodesAvailableError if poolsFull { - return failed(NoNodesAvailableError{}) + return failed(noCapacity(ShortageHugepagePool)) + } + if errors.As(err, &fromChoose) { + reason := fromChoose.Reason + if reason == ShortageUnknown { + reason = classifyShortage() + } + if fromChoose.PoolsFull { + poolsFull = true + } + return failed(noCapacity(reason)) } return failed(err) @@ -301,6 +369,9 @@ func placeSandbox( switch statusCode { case codes.ResourceExhausted: refusals++ + if ok { + lastRefusal = classifyRefusal(st) + } failedNode.PlacementMetrics.Skip(sbxRequest.GetSandbox().GetSandboxId()) if ok && createRefusedByStatus(st) { skipped[failedNode.ID] = struct{}{} @@ -353,6 +424,33 @@ func placeSandbox( return failed(SandboxCreateError{Attempts: attempt, LastErr: lastCreateErr}) } +// classifyRefusal labels a node's ResourceExhausted message so a capacity +// shortage carries what the node was short of, not just "no nodes available". +// It matches on the orchestrator's own wording; an unrecognised refusal is +// labelled "other" rather than dropped, so a reworded node message degrades +// to a coarse label instead of a blank one. +func classifyRefusal(st *status.Status) string { + if st == nil { + return "" + } + msg := st.Message() + switch { + case strings.Contains(msg, "max number of running sandboxes"): + return "running_cap" + case strings.Contains(msg, "too many sandboxes starting"): + return "starting_cap" + case strings.Contains(msg, "not enough memory to map the guest"): + return "guest_memory_stopping" + case strings.Contains(msg, "not accepting new sandboxes"): + return "node_not_accepting" + case strings.Contains(msg, "being migrated"), strings.Contains(msg, "rebooting"), + strings.Contains(msg, "being checkpointed"), strings.Contains(msg, "persisting sandbox"): + return "lifecycle_busy" + default: + return "other" + } +} + // createRefusedForGood reports a create the node refused with the typed // detail that says no other node and no retry changes the answer. func createRefusedForGood(st *status.Status) bool { diff --git a/packages/api/internal/orchestrator/placement/placement_best_of_K.go b/packages/api/internal/orchestrator/placement/placement_best_of_K.go index 4f9844f77d..d53d9bb37b 100644 --- a/packages/api/internal/orchestrator/placement/placement_best_of_K.go +++ b/packages/api/internal/orchestrator/placement/placement_best_of_K.go @@ -167,8 +167,11 @@ func (b *BestOfK) chooseNode(_ context.Context, nodes []*nodemanager.Node, exclu if bestNode == nil { // Full pools, or an unavailable node that would fit, are a shortage that // clears, not a mismatch. - if poolsFull || unavailableNodeWouldFit(nodes, excludedNodes, requirements, filterByLabels, requiredLabels) { - return nil, NoNodesAvailableError{} + if poolsFull { + return nil, NoNodesAvailableError{Reason: ShortageHugepagePool, PoolsFull: true} + } + if unavailableNodeWouldFit(nodes, excludedNodes, requirements, filterByLabels, requiredLabels) { + return nil, NoNodesAvailableError{Reason: ShortageNoCandidate} } return nil, FailedToPlaceSandboxError{