diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 82f6d32386..6d36e3861a 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -570,6 +570,18 @@ Key mechanisms (all under `pkg/sandbox/`): (with SNI/Host-inspecting TCP firewall for domain allow/deny lists). Slots are pooled and reused; slot indexes are allocated locally against the node's netns state (leftover namespaces from a previous run are torn down by startup reclaim). + **Guest IP stack**: guest networking is **IPv4-only by default**. The host tap has no IPv6 + router (no router advertisements, no routable prefix), so the guest boots with IPv6 addresses + stripped from every interface (`sysctl.net.ipv6.conf.{all,default}.disable_ipv6=1` on the + kernel cmdline). This leaves the kernel with no IPv6 source address, so address selection + (RFC 6724) prefers IPv4 and the ~250 ms Happy Eyeballs penalty (RFC 8305) — a runtime that + races AAAA first, e.g. Node's `autoSelectFamily`, then times out on the unroutable `fe80::` + link-local address — does not occur. Unlike `ipv6.disable=1`, the `AF_INET6` socket API and + `bind(::)` still work, so a dual-stack wildcard server in the guest still binds; only an + explicit `[::1]` loopback bind is unavailable. The switch is `Slot.HasIPv6Router()` → + `ProcessOptions.IPv6RouterConfigured` → the IPv6 kernel args; when the host networking layer + gains a full IPv6 stack (router + per-slot prefix), `HasIPv6Router()` returns true and guests + boot dual-stack (`ipv6.disable=0`, `ipv6.autoconf=1`) without further code changes. NFS release removes the old lifecycle's route, closes MOUNT admission and invalidates its handles before slot reuse. Physical root-descriptor cleanup runs independently and stays tracked by `nfs.retirements.gauge` until it finishes; a stuck backing operation does not diff --git a/packages/orchestrator/pkg/sandbox/fc/kernel_args.go b/packages/orchestrator/pkg/sandbox/fc/kernel_args.go index c38e82a481..9a1c594004 100644 --- a/packages/orchestrator/pkg/sandbox/fc/kernel_args.go +++ b/packages/orchestrator/pkg/sandbox/fc/kernel_args.go @@ -110,10 +110,10 @@ func buildKernelArgsFor(arch string, ipv4 string, options ProcessOptions) Kernel // Define kernel init path "init": options.InitScriptPath, - // Networking IPv4 and IPv6 - "ip": ipv4, - "ipv6.disable": "0", - "ipv6.autoconf": "1", + // Networking. IPv4 is always configured via ip=. The guest's IPv6 + // behaviour is set just below, derived from whether the host tap has an + // IPv6 router configured. + "ip": ipv4, // Wait 1 second before exiting FC after panic or reboot "panic": "1", @@ -129,6 +129,34 @@ func buildKernelArgsFor(arch string, ipv4 string, options ProcessOptions) Kernel "selinux": "0", } + // Guest IPv6. The host tap only has an IPv6 router when the slot says so + // (options.IPv6RouterConfigured, from Slot.HasIPv6Router). Without a router, + // SLAAC yields only an unroutable fe80:: link-local address; leaving IPv6 + // addresses on the guest makes some runtimes/libraries that run their own + // Happy Eyeballs (RFC 8305: Node's autoSelectFamily, curl, Go dual-stack + // dialing) try IPv6 first and wait out a ~250 ms timeout before falling + // back to IPv4 on every outbound dual-stack connection. + // + // So by default we strip IPv6 addresses from every interface via the + // per-stack sysctls, set on the kernel command line (supported since Linux + // 5.8; guest kernels are 5.10/6.1). Unlike ipv6.disable=1, this keeps the + // AF_INET6 socket API and `bind(::)` working — a dual-stack wildcard server + // in the guest still binds — it only removes the (unroutable) addresses so + // the kernel has no IPv6 source and address selection (RFC 6724) prefers + // IPv4. The one behaviour it drops is an explicit [::1] loopback bind, which + // has no routable counterpart without a v6 stack anyway. + // + // When the host networking layer gains a real IPv6 router + per-slot prefix, + // Slot.HasIPv6Router returns true and the guest keeps IPv6 enabled with + // SLAAC (ipv6.disable=0, ipv6.autoconf=1), yielding a routable address. + if options.IPv6RouterConfigured { + args["ipv6.disable"] = "0" + args["ipv6.autoconf"] = "1" + } else { + args["sysctl.net.ipv6.conf.all.disable_ipv6"] = "1" + args["sysctl.net.ipv6.conf.default.disable_ipv6"] = "1" + } + // The i8042 controller and kvm-clock exist only on x86; an arm64 guest has neither and // keeps the architected timer as its clocksource. The serial console is ttyS0 on both: // Firecracker exposes a 16550-compatible UART on arm64 too, and the arm64 guest kernel diff --git a/packages/orchestrator/pkg/sandbox/fc/kernel_args_test.go b/packages/orchestrator/pkg/sandbox/fc/kernel_args_test.go index 12fab5abf6..fe3c31955c 100644 --- a/packages/orchestrator/pkg/sandbox/fc/kernel_args_test.go +++ b/packages/orchestrator/pkg/sandbox/fc/kernel_args_test.go @@ -27,15 +27,17 @@ func TestBuildKernelArgs_DefaultIsPinned(t *testing.T) { name: "production defaults", options: ProcessOptions{InitScriptPath: "/sbin/init"}, want: "i8042.noaux i8042.nokbd init=/sbin/init ip=" + testIPv4 + - " ipv6.autoconf=1 ipv6.disable=0 loglevel=1 panic=1 pci=off quiet" + - " random.trust_cpu=on reboot=k rootflags=discard selinux=0", + " loglevel=1 panic=1 pci=off quiet" + + " random.trust_cpu=on reboot=k rootflags=discard selinux=0" + + " sysctl.net.ipv6.conf.all.disable_ipv6=1 sysctl.net.ipv6.conf.default.disable_ipv6=1", }, { name: "kvm clock", options: ProcessOptions{InitScriptPath: "/sbin/init", KvmClock: true}, want: "clocksource=kvm-clock i8042.noaux i8042.nokbd init=/sbin/init ip=" + testIPv4 + - " ipv6.autoconf=1 ipv6.disable=0 loglevel=1 panic=1 pci=off quiet" + - " random.trust_cpu=on reboot=k rootflags=discard selinux=0", + " loglevel=1 panic=1 pci=off quiet" + + " random.trust_cpu=on reboot=k rootflags=discard selinux=0" + + " sysctl.net.ipv6.conf.all.disable_ipv6=1 sysctl.net.ipv6.conf.default.disable_ipv6=1", }, { // Kernel logs drop `quiet` and raise the log level; asserted because the @@ -43,17 +45,30 @@ func TestBuildKernelArgs_DefaultIsPinned(t *testing.T) { name: "kernel logs", options: ProcessOptions{InitScriptPath: "/sbin/init", KernelLogs: true}, want: "console=ttyS0 i8042.noaux i8042.nokbd init=/sbin/init ip=" + testIPv4 + - " ipv6.autoconf=1 ipv6.disable=0 loglevel=5 panic=1 pci=off" + - " random.trust_cpu=on reboot=k rootflags=discard selinux=0", + " loglevel=5 panic=1 pci=off" + + " random.trust_cpu=on reboot=k rootflags=discard selinux=0" + + " sysctl.net.ipv6.conf.all.disable_ipv6=1 sysctl.net.ipv6.conf.default.disable_ipv6=1", }, { name: "systemd to kernel logs", options: ProcessOptions{InitScriptPath: "/sbin/init", SystemdToKernelLogs: true}, want: "console=ttyS0 i8042.noaux i8042.nokbd init=/sbin/init ip=" + testIPv4 + - " ipv6.autoconf=1 ipv6.disable=0 loglevel=5 panic=1 pci=off" + + " loglevel=5 panic=1 pci=off" + " random.trust_cpu=on reboot=k rootflags=discard selinux=0" + + " sysctl.net.ipv6.conf.all.disable_ipv6=1 sysctl.net.ipv6.conf.default.disable_ipv6=1" + " systemd.journald.forward_to_console", }, + { + // When the host tap has a real IPv6 router, the guest keeps IPv6 + // enabled with SLAAC (ipv6.disable=0, ipv6.autoconf=1) so it can + // obtain a routable address — and the address-stripping sysctls are + // absent. + name: "ipv6 router configured", + options: ProcessOptions{InitScriptPath: "/sbin/init", IPv6RouterConfigured: true}, + want: "i8042.noaux i8042.nokbd init=/sbin/init ip=" + testIPv4 + + " ipv6.autoconf=1 ipv6.disable=0 loglevel=1 panic=1 pci=off quiet" + + " random.trust_cpu=on reboot=k rootflags=discard selinux=0", + }, } for _, tt := range tests { @@ -80,22 +95,25 @@ func TestBuildKernelArgs_Arm64DropsX86OnlyParams(t *testing.T) { name: "production defaults", options: ProcessOptions{InitScriptPath: "/sbin/init"}, want: "init=/sbin/init ip=" + testIPv4 + - " ipv6.autoconf=1 ipv6.disable=0 loglevel=1 panic=1 pci=off quiet" + - " random.trust_cpu=on reboot=k rootflags=discard selinux=0", + " loglevel=1 panic=1 pci=off quiet" + + " random.trust_cpu=on reboot=k rootflags=discard selinux=0" + + " sysctl.net.ipv6.conf.all.disable_ipv6=1 sysctl.net.ipv6.conf.default.disable_ipv6=1", }, { name: "kvm clock requested is not applied", options: ProcessOptions{InitScriptPath: "/sbin/init", KvmClock: true}, want: "init=/sbin/init ip=" + testIPv4 + - " ipv6.autoconf=1 ipv6.disable=0 loglevel=1 panic=1 pci=off quiet" + - " random.trust_cpu=on reboot=k rootflags=discard selinux=0", + " loglevel=1 panic=1 pci=off quiet" + + " random.trust_cpu=on reboot=k rootflags=discard selinux=0" + + " sysctl.net.ipv6.conf.all.disable_ipv6=1 sysctl.net.ipv6.conf.default.disable_ipv6=1", }, { name: "kernel logs still go to ttyS0", options: ProcessOptions{InitScriptPath: "/sbin/init", KernelLogs: true}, want: "console=ttyS0 init=/sbin/init ip=" + testIPv4 + - " ipv6.autoconf=1 ipv6.disable=0 loglevel=5 panic=1 pci=off" + - " random.trust_cpu=on reboot=k rootflags=discard selinux=0", + " loglevel=5 panic=1 pci=off" + + " random.trust_cpu=on reboot=k rootflags=discard selinux=0" + + " sysctl.net.ipv6.conf.all.disable_ipv6=1 sysctl.net.ipv6.conf.default.disable_ipv6=1", }, } diff --git a/packages/orchestrator/pkg/sandbox/fc/process.go b/packages/orchestrator/pkg/sandbox/fc/process.go index d37c958a51..f0b2da7282 100644 --- a/packages/orchestrator/pkg/sandbox/fc/process.go +++ b/packages/orchestrator/pkg/sandbox/fc/process.go @@ -121,6 +121,14 @@ type ProcessOptions struct { // BootCpus caps the CPUs the guest kernel brings up (maxcpus=); 0 is all. Create sets it. BootCpus int64 + // IPv6RouterConfigured reports whether the host-side tap interface for this + // slot has an IPv6 router (router advertisements + a routable prefix). When + // false (the default) the guest boots with IPv6 addresses stripped from every + // interface, so the unroutable fe80:: link-local SLAAC address does not + // trigger the Happy Eyeballs ~250 ms fallback on dual-stack outbound + // connections. Set from Slot.HasIPv6Router() in Create. + IPv6RouterConfigured bool + // ACPIPowerOff declares the ACPI power-off device, so a guest power-off // ends the process instead of halting the vCPUs. A Firecracker without // the device refuses it, and the boot goes on without. @@ -553,6 +561,7 @@ func (p *Process) configureGuestDevices( // IPv4 configuration - format: [local_ip]::[gateway_ip]:[netmask]:hostname:iface:dhcp_option:[dns] ipv4 := fmt.Sprintf("%s::%s:%s:instance:%s:off:%s", p.slot.NamespaceIP(), p.slot.TapIPString(), p.slot.TapMaskString(), p.slot.VpeerName(), p.slot.TapName()) + options.IPv6RouterConfigured = p.slot.HasIPv6Router() kernelArgs := buildKernelArgs(ipv4, options).String() if boot.NoKernelArgs { kernelArgs = "" diff --git a/packages/orchestrator/pkg/sandbox/network/slot.go b/packages/orchestrator/pkg/sandbox/network/slot.go index 07affff516..95e76a7019 100644 --- a/packages/orchestrator/pkg/sandbox/network/slot.go +++ b/packages/orchestrator/pkg/sandbox/network/slot.go @@ -309,6 +309,21 @@ func (s *Slot) TapMAC() string { return tapMAC } +// HasIPv6Router reports whether the host-side tap interface for this slot has +// an IPv6 router configured — i.e. it sends router advertisements so the guest +// can obtain a routable IPv6 address via SLAAC. +// +// Today the host networking stack is IPv4-only: no RA is sent on the tap, so +// SLAAC would produce only an unroutable fe80:: link-local address inside the +// guest. Returning false here makes the orchestrator boot guests with IPv6 +// addresses stripped (see buildKernelArgs), eliminating the ~250 ms Happy +// Eyeballs penalty an unroutable IPv6 address causes. When the host networking +// layer gains a full IPv6 stack (router + per-slot prefix), flip this to true +// (or derive it from the slot's real IPv6 state). +func (s *Slot) HasIPv6Router() bool { + return false +} + // UseNonDNSUDPMode sets the provider every later chain build consults. func (s *Slot) UseNonDNSUDPMode(provider NonDNSUDPModeProvider) { s.nonDNSUDPMode = provider