feat(infra): rank Kubernetes workloads by replica health and restarts

apps/ai/src/mcp/__evals__/infra-world.ts

@@ -545,6 +545,17 @@ const workloadRows = (sql: string) => {
 			avgCpuLimitPct: mean((s) => s.cpuLimitPct),
 			avgMemoryLimitPct: mean((s) => s.memoryLimitPct),
 			avgCpuUsage: mean((s) => s.cpuUsage),
+			peakCpuLimitPct: Math.max(...stats.map((s) => s.cpuLimitPctPeak)),
+			peakMemoryLimitPct: Math.max(...stats.map((s) => s.memoryLimitPctPeak)),
+			saturation: Math.max(...stats.map((s) => Math.max(s.cpuLimitPctPeak, s.memoryLimitPctPeak))),
+			cpuUsage: stats.reduce((sum, s) => sum + s.cpuUsage, 0),
+			desiredReplicas: pods.length,
+			readyReplicas: pods.length,
+			replicaSamples: 1,
+			shortfall: 0,
+			failedPods: 0,
+			restarts: 0,
+			lastTerminatedReason: "",
 		}
 	})
 }

apps/ai/src/mcp/lib/infra.ts

@@ -8,8 +8,8 @@ import { CH, formatWarehouseDateTime, parseWarehouseDateTime } from "@maple/quer
 import { WarehouseExecutor } from "@maple/query-engine/observability"
 import type { InfraEntityRow, InfraKind, InfraKindSection, InfraSeriesStats } from "@maple/domain/mcp-outputs"
 
-export type WorkloadKind = "deployment" | "statefulset" | "daemonset"
-export const WORKLOAD_KINDS: ReadonlyArray<WorkloadKind> = ["deployment", "statefulset", "daemonset"]
+export type WorkloadKind = "deployment" | "statefulset" | "daemonset" | "job"
+export const WORKLOAD_KINDS: ReadonlyArray<WorkloadKind> = ["deployment", "statefulset", "daemonset", "job"]
 
 export interface InfraWindow {
 	readonly startTime: string
@@ -225,11 +225,11 @@ const workloadRow = (kind: WorkloadKind, row: CH.ListWorkloadsOutput): InfraEnti
 	workloadKind: kind,
 	...placement(row),
 	podCount: row.podCount,
-	cpuCores: row.avgCpuUsage,
-	cpu: row.avgCpuLimitPct,
-	memory: row.avgMemoryLimitPct,
-	saturation: Math.max(row.avgCpuLimitPct, row.avgMemoryLimitPct),
-	...(row.avgCpuLimitPct === 0 && row.avgMemoryLimitPct === 0 ? { unbounded: true } : undefined),
+	cpuCores: row.cpuUsage,
+	cpu: row.peakCpuLimitPct,
+	memory: row.peakMemoryLimitPct,
+	saturation: row.saturation,
+	...(row.peakCpuLimitPct === 0 && row.peakMemoryLimitPct === 0 ? { unbounded: true } : undefined),
 })
 
 const containerRow = (row: CH.ListContainersOutput): InfraEntityRow => ({

apps/api/src/routes/internal/query-engine.http.ts

@@ -1847,9 +1847,14 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleInternalApi, "query
 								environment: row.environment,
 								podCount: row.podCount || 0,
 								lastSeen: DateTime.formatIso(row.lastSeen),
-								avgCpuLimitPct: row.avgCpuLimitPct || 0,
-								avgMemoryLimitPct: row.avgMemoryLimitPct || 0,
-								avgCpuUsage: row.avgCpuUsage || 0,
+								desiredReplicas: row.replicaSamples > 0 ? row.desiredReplicas : null,
+								readyReplicas: row.replicaSamples > 0 ? row.readyReplicas : null,
+								failedPods: row.failedPods || 0,
+								restarts: row.restarts || 0,
+								lastTerminatedReason: row.lastTerminatedReason,
+								peakCpuLimitPct: row.peakCpuLimitPct || 0,
+								peakMemoryLimitPct: row.peakMemoryLimitPct || 0,
+								cpuUsage: row.cpuUsage || 0,
 							})),
 						})
 					}),

apps/web/src/api/warehouse/infra.ts

@@ -50,7 +50,7 @@ import { Effect } from "effect"
 import { MapleInternalAtomClient } from "@/lib/services/common/internal-atom-client"
 import { runWarehouseQuery } from "./effect-utils"
 
-export type WorkloadKind = "deployment" | "statefulset" | "daemonset"
+export type WorkloadKind = "deployment" | "statefulset" | "daemonset" | "job"
 
 /** Mirrors PodSortKeyLiteral in @maple/domain — `saturation` is peak-of-either-limit. */
 export type PodSortKey = "saturation" | "cpuUsage" | "cpuLimitPct" | "memoryLimitPct" | "podName" | "lastSeen"

apps/web/src/components/infra/workload-table.tsx

@@ -2,13 +2,16 @@ import { Link } from "@tanstack/react-router"
 import { TruncatedText } from "@maple/ui/components/ui/truncated-text"
 
 import { Skeleton } from "@maple/ui/components/ui/skeleton"
+import { cn } from "@maple/ui/lib/utils"
+import { TONE_TEXT, type Tone } from "@maple/ui/lib/tone"
 
 import type { ListWorkloadsResponse } from "@maple/domain/http"
 import type { WorkloadKind } from "@/api/warehouse/infra"
 
 import { HostStatusBadge } from "./status-badge"
 import { MeterRows } from "./primitives/meter-rows"
 import { MetaLine } from "./primitives/meta-line"
+import { restartTone } from "./pod-restarts"
 import {
 	ColumnHead,
 	DataTable,
@@ -20,7 +23,13 @@ import { RelativeTime } from "@/components/common/relative-time"
 
 export type WorkloadRow = ListWorkloadsResponse["data"][number]
 
-type SortKey = "workloadName" | "namespace" | "podCount" | "avgCpuLimitPct" | "avgMemoryLimitPct" | "lastSeen"
+type SortKey =
+	| "workloadName"
+	| "podCount"
+	| "restarts"
+	| "peakCpuLimitPct"
+	| "peakMemoryLimitPct"
+	| "lastSeen"
 
 interface WorkloadTableProps {
 	workloads: ReadonlyArray<WorkloadRow>
@@ -30,7 +39,7 @@ interface WorkloadTableProps {
 }
 
 /** Declared once and rendered by both the table and its skeleton so widths cannot drift. */
-function WorkloadColumns({ sort }: { sort?: SortControls<SortKey> }) {
+function WorkloadColumns({ sort, kind }: { sort?: SortControls<SortKey>; kind?: WorkloadKind }) {
 	return (
 		<>
 			<ColumnHead<SortKey>
@@ -39,21 +48,29 @@ function WorkloadColumns({ sort }: { sort?: SortControls<SortKey> }) {
 				{...sort}
 				width="w-0 flex-1 min-w-[260px]"
 			/>
+			<ColumnHead<SortKey> label={kind === "job" ? "Completions" : "Ready"} width="w-[110px]" />
 			<ColumnHead<SortKey> label="Pods" sortKey="podCount" {...sort} align="right" width="w-[60px]" />
 			<ColumnHead<SortKey>
-				label="Avg CPU"
-				sortKey="avgCpuLimitPct"
+				label="Restarts"
+				sortKey="restarts"
 				{...sort}
 				align="right"
-				width="w-[160px]"
+				width="w-[140px]"
+			/>
+			<ColumnHead<SortKey>
+				label="CPU peak"
+				sortKey="peakCpuLimitPct"
+				{...sort}
+				align="right"
+				width="w-[140px]"
 				hidden="hidden md:flex"
 			/>
 			<ColumnHead<SortKey>
-				label="Avg memory"
-				sortKey="avgMemoryLimitPct"
+				label="Memory peak"
+				sortKey="peakMemoryLimitPct"
 				{...sort}
 				align="right"
-				width="w-[160px]"
+				width="w-[140px]"
 				hidden="hidden lg:flex"
 			/>
 			<ColumnHead<SortKey>
@@ -78,25 +95,59 @@ export function WorkloadTableLoading() {
 					<Skeleton className="h-4 w-48" />
 					<Skeleton className="mt-1.5 h-3 w-32" />
 				</div>
+				<Skeleton className="h-3 w-[110px]" />
 				<Skeleton className="h-3 w-[60px]" />
-				<Skeleton className="hidden h-3 w-[160px] md:block" />
-				<Skeleton className="hidden h-3 w-[160px] lg:block" />
+				<Skeleton className="h-3 w-[140px]" />
+				<Skeleton className="hidden h-3 w-[140px] md:block" />
+				<Skeleton className="hidden h-3 w-[140px] lg:block" />
 				<Skeleton className="h-3 w-[100px]" />
 			</DataTable.SkeletonRows>
 		</DataTable.Root>
 	)
 }
 
+/**
+ * Ready against desired, from the cluster collector. A job counts completions, so a
+ * running job is not "short"; its failed pods are what reads as trouble.
+ */
+function ReplicaCell({ workload, kind }: { workload: WorkloadRow; kind: WorkloadKind }) {
+	const { desiredReplicas: desired, readyReplicas: ready, failedPods } = workload
+	if (desired === null || ready === null) {
+		return <span className="text-muted-foreground">—</span>
+	}
+	const tone: Tone =
+		kind === "job"
+			? failedPods > 0
+				? "crit"
+				: "neutral"
+			: ready === 0 && desired > 0
+				? "crit"
+				: ready < desired
+					? "warn"
+					: "neutral"
+	return (
+		<span className={TONE_TEXT[tone]}>
+			{ready} / {desired}
+			{kind === "job" && failedPods > 0 && ` · ${failedPods} failed`}
+		</span>
+	)
+}
+
 export function WorkloadTable({ workloads, kind, waiting, referenceTime }: WorkloadTableProps) {
+	// No initial key: the server already sorts worst first (missing replicas, failures, restarts, saturation).
 	const { sorted, sortKey, sortDir, handleSort } = useTableSort<WorkloadRow, SortKey>(workloads, {
-		initialKey: "avgCpuLimitPct",
-		stringKeys: ["workloadName", "namespace"],
+		initialKey: null,
+		stringKeys: ["workloadName"],
+		resettable: true,
 	})
 
 	return (
 		<DataTable.Root ariaLabel="Workloads" waiting={waiting}>
 			<DataTable.Head>
-				<WorkloadColumns sort={{ currentKey: sortKey, dir: sortDir, onSort: handleSort }} />
+				<WorkloadColumns
+					kind={kind}
+					sort={{ currentKey: sortKey, dir: sortDir, onSort: handleSort }}
+				/>
 			</DataTable.Head>
 			{sorted.length === 0 && <DataTable.Empty>No workloads match your filter.</DataTable.Empty>}
 
@@ -119,14 +170,28 @@ export function WorkloadTable({ workloads, kind, waiting, referenceTime }: Workl
 						</div>
 						<MetaLine items={[wl.namespace && `ns ${wl.namespace}`, `kind ${kind}`]} />
 					</div>
+					<div className="w-[110px] font-mono text-xs tabular-nums">
+						<ReplicaCell workload={wl} kind={kind} />
+					</div>
 					<div className="w-[60px] text-right font-mono text-xs tabular-nums text-foreground/80">
 						{wl.podCount.toLocaleString()}
 					</div>
-					<div className="hidden w-[160px] md:block">
-						<MeterRows hideLabels meters={[{ label: "CPU", fraction: wl.avgCpuLimitPct }]} />
+					<div
+						className={cn(
+							"flex w-[140px] flex-col items-end font-mono text-xs tabular-nums",
+							TONE_TEXT[restartTone(wl.restarts, wl.lastTerminatedReason)],
+						)}
+					>
+						<span>{wl.restarts.toLocaleString()}</span>
+						{wl.restarts > 0 && wl.lastTerminatedReason && (
+							<span className="text-2xs text-muted-foreground">{wl.lastTerminatedReason}</span>
+						)}
+					</div>
+					<div className="hidden w-[140px] md:block">
+						<MeterRows hideLabels meters={[{ label: "CPU", fraction: wl.peakCpuLimitPct }]} />
 					</div>
-					<div className="hidden w-[160px] lg:block">
-						<MeterRows hideLabels meters={[{ label: "MEM", fraction: wl.avgMemoryLimitPct }]} />
+					<div className="hidden w-[140px] lg:block">
+						<MeterRows hideLabels meters={[{ label: "MEM", fraction: wl.peakMemoryLimitPct }]} />
 					</div>
 					<div className="w-[100px] text-right">
 						<RelativeTime

apps/web/src/routes/infra/kubernetes/workloads/$kind/$workloadName.tsx

@@ -43,7 +43,7 @@ const workloadDetailSearchSchema = Schema.Struct({
 	...TimeRangeSearchFields,
 })
 
-const WorkloadKindSchema = Schema.Literals(["deployment", "statefulset", "daemonset"])
+const WorkloadKindSchema = Schema.Literals(["deployment", "statefulset", "daemonset", "job"])
 
 const paramsSchema = Schema.Struct({
 	kind: WorkloadKindSchema,
@@ -70,6 +70,7 @@ const KIND_LABEL: Record<WorkloadKind, string> = {
 	deployment: "Deployment",
 	statefulset: "StatefulSet",
 	daemonset: "DaemonSet",
+	job: "Job",
 } satisfies Record<WorkloadKind, string>
 
 function WorkloadDetailPage() {

apps/web/src/routes/infra/kubernetes/workloads/index.tsx

@@ -34,10 +34,11 @@ import { sessionTimeRangeSearchMiddleware } from "@/components/time-range-picker
 import { TONE_FILL } from "@maple/ui/lib/tone"
 
 const DEFAULT_PRESET = "12h"
-// The list query returns the most recently seen N workloads; the toolbar says so at the cap.
+// The list query returns the N worst workloads (missing replicas, failures, restarts, saturation);
+// the toolbar says so at the cap.
 const WORKLOAD_LIST_LIMIT = 200
 
-const WorkloadKindLiteral = Schema.Literals(["deployment", "statefulset", "daemonset"])
+const WorkloadKindLiteral = Schema.Literals(["deployment", "statefulset", "daemonset", "job"])
 
 /** A one-click scope from the band. `undefined` means every workload. */
 type WorkloadScope = "saturated" | "elevated" | "stale"
@@ -68,17 +69,13 @@ const KIND_OPTIONS = [
 	{ value: "deployment", label: "Deployments" },
 	{ value: "statefulset", label: "StatefulSets" },
 	{ value: "daemonset", label: "DaemonSets" },
+	{ value: "job", label: "Jobs" },
 ] as const satisfies ReadonlyArray<{ value: WorkloadKind; label: string }>
 
-/**
- * The workload list is average-over-window, not peak: the query aggregates per
- * workload and only carries the mean. So the band says "average", and a
- * workload that spikes without moving its mean stays out of the red here — the
- * pods view is where a spike shows.
- */
+/** Saturation is the worst pod's peak, so one pod at its limit marks the whole workload. */
 function scopeOf(workload: WorkloadRow, referenceTime: string): WorkloadScope | null {
 	if (deriveHostStatus(workload.lastSeen, referenceTime) === "ended") return "stale"
-	const level = severityLevel(Math.max(workload.avgCpuLimitPct, workload.avgMemoryLimitPct))
+	const level = severityLevel(Math.max(workload.peakCpuLimitPct, workload.peakMemoryLimitPct))
 	if (level === "crit") return "saturated"
 	if (level === "warn") return "elevated"
 	return null
@@ -203,7 +200,7 @@ function WorkloadsPage() {
 							<InfraSetupEmpty
 								icon={<GridIcon size={16} />}
 								title={`No ${kindOption.label} reporting yet`}
-								description="Maple aggregates pod metrics by k8s.deployment.name, k8s.statefulset.name, and k8s.daemonset.name. If the Helm chart is not installed yet, install it so the k8sattributes processor can tag pod metrics with their workload. If it is, try another workload kind."
+								description="Maple aggregates pod metrics by k8s.deployment.name, k8s.statefulset.name, k8s.daemonset.name, and k8s.job.name. If the Helm chart is not installed yet, install it so the k8sattributes processor can tag pod metrics with their workload. If it is, try another workload kind."
 								installTab="kubernetes"
 								actionLabel="Install the Helm chart"
 								docs="kubernetes"
@@ -230,14 +227,14 @@ function WorkloadsPage() {
 						{
 							scope: "saturated",
 							label: "Saturated",
-							hint: "avg ≥90%",
+							hint: "a pod ≥90%",
 							value: saturated,
 							tone: "crit",
 						},
 						{
 							scope: "elevated",
 							label: "Elevated",
-							hint: "avg ≥60%",
+							hint: "a pod ≥60%",
 							value: elevated,
 							tone: "warn",
 						},
@@ -264,7 +261,7 @@ function WorkloadsPage() {
 							<FleetBand
 								total={workloads.length}
 								noun={kindOption.label.slice(0, -1).toLowerCase()}
-								caption="share of the fleet by average utilization"
+								caption="share of the fleet by its busiest pod's peak"
 								segments={[
 									{ key: "healthy", count: healthy, className: TONE_FILL.ok },
 									{
@@ -290,7 +287,7 @@ function WorkloadsPage() {
 									placeholder="Search workloads…"
 									trailing={
 										workloads.length >= WORKLOAD_LIST_LIMIT
-											? `${formatNumber(filtered.length)} of the ${formatNumber(WORKLOAD_LIST_LIMIT)} most recently seen workloads`
+											? `${formatNumber(filtered.length)} of the ${formatNumber(WORKLOAD_LIST_LIMIT)} least healthy workloads`
 											: countLabel(filtered.length, filtered.length, "workload")
 									}
 								>