From e19b6b72297963b429c5ac5fcbd55016144d85d8 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Sat, 12 Sep 2026 22:21:10 +0000 Subject: [PATCH 01/24] multikernel: simplify and harden duplex IPI transport Replace the owner-and-slot-state producer protocol with one spinlocked producer per duplex direction and one release/acquire ready flag per slot. Reset a parent/child link only after the previous child is confirmed parked, and close endpoints before their backing image pages are released. Carry the parent and doorbell identities in the shared link, restore them through the boot device tree design, and route nested console output through the actual parent instance. Fail incompatible boot contexts and legacy image notes with explicit errors while keeping the vmlinux descriptor as the original u64 entry. Preserve host timer calibration for spawned x86 kernels and declare the OF dependency required by the upstream boot-tree manifest. Signed-off-by: Nikolay Nikolaev --- arch/x86/boot/header.S | 8 +- arch/x86/include/asm/multikernel.h | 5 +- arch/x86/include/uapi/asm/bootparam.h | 1 + arch/x86/kernel/kexec-bzimage64.c | 24 ++ arch/x86/kernel/kexec-vmlinux.c | 66 +++- arch/x86/kernel/platform-quirks.c | 4 +- arch/x86/multikernel/head_64.S | 3 +- arch/x86/multikernel/spawn.c | 133 +++++-- drivers/tty/mktty.c | 39 ++- include/linux/multikernel.h | 101 ++++-- include/linux/multikernel_abi.h | 9 + kernel/kexec_core.c | 65 ++-- kernel/multikernel/Kconfig | 1 + kernel/multikernel/core.c | 121 +++++-- kernel/multikernel/cpuset.c | 271 ++++++++++++--- kernel/multikernel/hotplug.c | 14 + kernel/multikernel/instance_dt.c | 129 +++++-- kernel/multikernel/internal.h | 9 + kernel/multikernel/ipi.c | 482 ++++++++++++-------------- kernel/multikernel/manifest.c | 14 +- 20 files changed, 1021 insertions(+), 478 deletions(-) create mode 100644 include/linux/multikernel_abi.h diff --git a/arch/x86/boot/header.S b/arch/x86/boot/header.S index 9bea5a1e2c52cb..6758247c93fd7e 100644 --- a/arch/x86/boot/header.S +++ b/arch/x86/boot/header.S @@ -379,7 +379,13 @@ xloadflags: #define XLF56 0 #endif - .word XLF0 | XLF1 | XLF23 | XLF4 | XLF56 +#ifdef CONFIG_MULTIKERNEL +# define XLF_MK XLF_MULTIKERNEL_IPI +#else +# define XLF_MK 0 +#endif + + .word XLF0 | XLF1 | XLF23 | XLF4 | XLF56 | XLF_MK cmdline_size: .long COMMAND_LINE_SIZE-1 #length of the command line, #added with boot protocol diff --git a/arch/x86/include/asm/multikernel.h b/arch/x86/include/asm/multikernel.h index 6f55eab4b35256..953fe71fc044a7 100644 --- a/arch/x86/include/asm/multikernel.h +++ b/arch/x86/include/asm/multikernel.h @@ -127,7 +127,7 @@ struct mk_spawn_context { u32 target_apic_id; /* Target CPU's APIC ID */ u32 flags; /* MK_SPAWN_F_* flags */ u32 ready; /* Signal flag */ - u32 reserved; /* Padding for alignment */ + u32 abi_magic; /* Host/spawn generation marker */ /* Keep all existing context offsets unchanged. */ struct boot_params bp; /* Standard x86 boot params */ /* Optional boot data belongs after boot_params, in the zeroed tail. */ @@ -198,7 +198,8 @@ int mk_spawn_cpu(struct mk_instance *instance, int cpu, /* The pool park set (park page, slot, page tables) as base,size pairs */ int mk_pool_park_regions(u64 *pairs, int max); -/* Initialize boot context tracking in spawn kernel */ +/* Validate and initialize boot context tracking in spawn kernel */ +struct mk_spawn_context *mk_validate_boot_context(phys_addr_t ctx_phys); void mk_init_boot_context(phys_addr_t ctx_phys); diff --git a/arch/x86/include/uapi/asm/bootparam.h b/arch/x86/include/uapi/asm/bootparam.h index c70be687a3ecc7..7099f7cd167dce 100644 --- a/arch/x86/include/uapi/asm/bootparam.h +++ b/arch/x86/include/uapi/asm/bootparam.h @@ -25,6 +25,7 @@ #define XLF_5LEVEL (1<<5) #define XLF_5LEVEL_ENABLED (1<<6) #define XLF_MEM_ENCRYPTION (1<<7) +#define XLF_MULTIKERNEL_IPI 0x0100 #ifndef __ASSEMBLER__ diff --git a/arch/x86/kernel/kexec-bzimage64.c b/arch/x86/kernel/kexec-bzimage64.c index 1002fbcf2f5d0b..1341501952d81b 100644 --- a/arch/x86/kernel/kexec-bzimage64.c +++ b/arch/x86/kernel/kexec-bzimage64.c @@ -554,6 +554,11 @@ static void *bzImage64_load(struct kimage *image, char *kernel, .buf_max = ULONG_MAX, .top_down = true }; header = (struct setup_header *)(kernel + setup_hdr_offset); + if (image->type == KEXEC_TYPE_MULTIKERNEL && + !(header->xloadflags & XLF_MULTIKERNEL_IPI)) { + pr_err("Loaded kernel lacks the required shared transport layout\n"); + return ERR_PTR(-EPROTONOSUPPORT); + } setup_sects = header->setup_sects; if (setup_sects == 0) setup_sects = 4; @@ -747,10 +752,29 @@ static void *bzImage64_load(struct kimage *image, char *kernel, /* For multikernel, setup custom e820 map */ if (image->type == KEXEC_TYPE_MULTIKERNEL) { +#ifdef CONFIG_MULTIKERNEL image->arch.mk_boot_params = bootparam_load_addr; + + /* + * setup_boot_parameters() copies the host subarchitecture. A + * spawn kernel must take the multikernel platform path instead. + */ + params->hdr.hardware_subarch = X86_SUBARCH_MULTIKERNEL; + + /* + * The spawn trampoline enters the compressed kernel directly, + * bypassing purgatory. The x86 boot protocol's 64-bit entry is + * 0x200 bytes from the start of the protected-mode payload. + */ + image->arch.mk_kernel_entry = kernel_load_addr + 0x200; + ret = mk_e820_fill(image->mk_instance, params); if (ret) goto out_free_params; +#else + ret = -EOPNOTSUPP; + goto out_free_params; +#endif } /* Allocate loader specific data */ diff --git a/arch/x86/kernel/kexec-vmlinux.c b/arch/x86/kernel/kexec-vmlinux.c index a7d31a722f33b7..ef5fc7ce6941b1 100644 --- a/arch/x86/kernel/kexec-vmlinux.c +++ b/arch/x86/kernel/kexec-vmlinux.c @@ -62,12 +62,16 @@ struct elf_kernel_info { /* * Find multikernel entry point from PT_NOTE section. - * Looks for note with name "Linux" and type 0x4d4b ('MK'). + * The note type carries the generation; the descriptor remains one u64. */ -static unsigned long find_multikernel_entry_note(const void *buf, size_t len, - const Elf64_Ehdr *ehdr) +#define MK_VMLINUX_LEGACY_NOTE_TYPE 0x4d4b + +static int find_multikernel_entry_note(const void *buf, size_t len, + const Elf64_Ehdr *ehdr, + unsigned long *entry) { const Elf64_Phdr *phdrs = buf + ehdr->e_phoff; + bool legacy = false; int i; for (i = 0; i < ehdr->e_phnum; i++) { @@ -91,25 +95,34 @@ static unsigned long find_multikernel_entry_note(const void *buf, size_t len, if (ptr + note_size > end) break; - if (nhdr->n_type == 0x4d4b && - nhdr->n_namesz == 6 && + if (nhdr->n_namesz == 6 && nhdr->n_descsz == sizeof(u64) && !memcmp(ptr + sizeof(*nhdr), "Linux", 6)) { - u64 entry = *(u64 *)(ptr + sizeof(*nhdr) + - ALIGN(nhdr->n_namesz, 4)); - pr_info("multikernel: entry=0x%llx\n", entry); - return entry; + const u64 *note_entry; + + note_entry = ptr + sizeof(*nhdr) + + ALIGN(nhdr->n_namesz, 4); + if (nhdr->n_type == MK_VMLINUX_NOTE_TYPE) { + *entry = *note_entry; + pr_info("multikernel: entry=0x%llx\n", + *note_entry); + return 0; + } + if (nhdr->n_type == MK_VMLINUX_LEGACY_NOTE_TYPE) + legacy = true; } ptr += note_size; } } - return 0; + return legacy ? -EPROTONOSUPPORT : -ENOENT; } /* * Parse ELF kernel and extract key information */ -static int kexec_parse_elf_kernel(const void *kernel_buf, unsigned long kernel_len, +static int kexec_parse_elf_kernel(const void *kernel_buf, + unsigned long kernel_len, + bool multikernel, struct elf_kernel_info *info) { const Elf64_Ehdr *ehdr; @@ -159,14 +172,25 @@ static int kexec_parse_elf_kernel(const void *kernel_buf, unsigned long kernel_l * PT_NOTE contains physical offset from load base, not virtual address. * This is the canonical way and survives symbol stripping. */ - info->multikernel_entry = find_multikernel_entry_note(kernel_buf, kernel_len, ehdr); - if (!info->multikernel_entry) { - pr_err("multikernel_startup_64 entry offset not found in PT_NOTE\n"); - return -ENOEXEC; + info->multikernel_entry = 0; + if (multikernel) { + int ret; + + ret = find_multikernel_entry_note(kernel_buf, kernel_len, ehdr, + &info->multikernel_entry); + if (ret == -EPROTONOSUPPORT) + pr_err("legacy vmlinux note type 0x%x is incompatible; expected 0x%x\n", + MK_VMLINUX_LEGACY_NOTE_TYPE, MK_VMLINUX_NOTE_TYPE); + else if (ret) + pr_err("multikernel ABI note type 0x%x not found\n", + MK_VMLINUX_NOTE_TYPE); + if (ret) + return ret == -ENOENT ? -ENOEXEC : ret; + + pr_info("Multikernel entry offset: 0x%lx\n", + info->multikernel_entry); } - pr_info("Multikernel entry offset: 0x%lx\n", info->multikernel_entry); - /* Find lowest load address and calculate total memory needed */ phdr = (const Elf64_Phdr *)(kernel_buf + ehdr->e_phoff); for (i = 0; i < ehdr->e_phnum; i++) { @@ -326,12 +350,13 @@ static void *vmlinux_load(struct kimage *image, char *kernel, .top_down = true }; struct kexec_buf pbuf = { .image = image, .buf_min = MIN_PURGATORY_ADDR, .buf_max = ULONG_MAX, .top_down = true }; + bool multikernel = image->type == KEXEC_TYPE_MULTIKERNEL; int ret; pr_info("Loading ELF vmlinux (type=%d)\n", image->type); /* Parse ELF headers */ - ret = kexec_parse_elf_kernel(kernel, kernel_len, &elf_info); + ret = kexec_parse_elf_kernel(kernel, kernel_len, multikernel, &elf_info); if (ret) { pr_err("Failed to parse ELF kernel: %d\n", ret); return ERR_PTR(ret); @@ -531,12 +556,17 @@ static void *vmlinux_load(struct kimage *image, char *kernel, /* For multikernel, setup custom e820 map */ if (image->type == KEXEC_TYPE_MULTIKERNEL) { +#ifdef CONFIG_MULTIKERNEL ret = mk_e820_fill(image->mk_instance, params); if (ret) { kvfree(ldata->kernel_buf); kfree(ldata); goto out_free_params; } +#else + ret = -EOPNOTSUPP; + goto out_free_params; +#endif } ldata->bootparams_buf = params; diff --git a/arch/x86/kernel/platform-quirks.c b/arch/x86/kernel/platform-quirks.c index 5d4705d407b4c4..fcd674b933102c 100644 --- a/arch/x86/kernel/platform-quirks.c +++ b/arch/x86/kernel/platform-quirks.c @@ -47,9 +47,9 @@ static void __init multikernel_setup_calibration(void) { phys_addr_t ctx_phys = orig_boot_params - offsetof(struct mk_spawn_context, bp); - struct mk_spawn_context *ctx = __va(ctx_phys); + struct mk_spawn_context *ctx = mk_validate_boot_context(ctx_phys); - if (ctx->self_phys != ctx_phys || !ctx->boot_tsc_khz) + if (!ctx || !ctx->boot_tsc_khz) return; multikernel_tsc_khz = ctx->boot_tsc_khz; diff --git a/arch/x86/multikernel/head_64.S b/arch/x86/multikernel/head_64.S index 3784147fd82f62..54890d936e615e 100644 --- a/arch/x86/multikernel/head_64.S +++ b/arch/x86/multikernel/head_64.S @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -269,7 +270,7 @@ SYM_CODE_END(multikernel_secondary_startup) .balign 4 .long 2f - 1f .long 4f - 3f - .long 0x4d4b + .long MK_VMLINUX_NOTE_TYPE 1: .asciz "Linux" 2: .balign 4 3: .quad multikernel_startup_64 - __START_KERNEL_map diff --git a/arch/x86/multikernel/spawn.c b/arch/x86/multikernel/spawn.c index c8741c95c1529e..78d9122761ff5b 100644 --- a/arch/x86/multikernel/spawn.c +++ b/arch/x86/multikernel/spawn.c @@ -72,6 +72,7 @@ /* Set in spawn kernels: the context this kernel booted from */ static struct mk_spawn_context *mk_boot_context; +static phys_addr_t mk_boot_context_phys; /* * This kernel's own context: where its CPUs park and wake. A spawn's @@ -86,6 +87,19 @@ static struct mk_spawn_context *mk_own_context(void) return mk_pool ? mk_pool->arch.slot : NULL; } +static void *spawn_trampoline_va; +static unsigned long spawn_trampoline_phys; +static bool spawn_trampoline_prepared; +static bool spawn_pool_park_prepared; +static bool spawn_park_ready; +static int spawn_park_error; + +bool mk_arch_park_ready(void) +{ + /* Pair with publication after both executable park mappings succeed. */ + return smp_load_acquire(&spawn_park_ready); +} + extern char multikernel_relocate_kernel_start[]; extern char multikernel_relocate_kernel_end[]; extern char mk_secondary_trampoline[]; @@ -455,6 +469,7 @@ int mk_arch_spawn_instance(struct kimage *image, struct mk_instance *instance, instance->arch.spawn_ctx->boot_tsc_khz = tsc_khz; instance->arch.spawn_ctx->boot_apic_hz = (unsigned long)lapic_timer_period * HZ; + instance->arch.spawn_ctx->abi_magic = MK_BOOT_CONTEXT_MAGIC; return mk_spawn_cpu(instance, cpu, instance->arch.spawn_ctx); } @@ -677,18 +692,38 @@ void __init mk_arch_register_cpu(u64 phys_id) topology_register_apic((u32)phys_id, CPU_ACPIID_INVALID, true); } -/* - * Initialize boot context tracking in spawn kernel. - * Called early during spawn kernel boot. - */ -void mk_init_boot_context(phys_addr_t ctx_phys) +static __noreturn void mk_reject_spawn_context(void) +{ + /* + * The host context layout is unknown, so neither its park state nor any + * shared context field is safe to use. Keep this CPU local and inert. An NMI + * can wake HLT, but returns to this loop with maskable interrupts still + * disabled; disable them again before every halt for defense in depth. + */ + for (;;) { + native_irq_disable(); + native_halt(); + } +} + +struct mk_spawn_context *mk_validate_boot_context(phys_addr_t ctx_phys) { struct mk_spawn_context *ctx; + phys_addr_t stamped_phys; + u32 abi_magic; if (!ctx_phys) { pr_err("mk_spawn: Boot context physical address is 0!\n"); - return; + return NULL; } + if (mk_boot_context) { + if (ctx_phys != mk_boot_context_phys) + mk_reject_spawn_context(); + return mk_boot_context; + } + /* Reject an invalid derived address before mapping or dereferencing it. */ + if (!IS_ALIGNED(ctx_phys, PAGE_SIZE)) + mk_reject_spawn_context(); /* * The spawn context is in the multikernel pool which is regular RAM, @@ -705,14 +740,28 @@ void mk_init_boot_context(phys_addr_t ctx_phys) * work and then fails much later, when this kernel shuts down and * its CPUs park on nonsense addresses. */ - if (ctx->self_phys != ctx_phys) { - pr_err("mk_spawn: Boot context at %pa is stamped %pa\n", - &ctx_phys, &ctx->self_phys); - pr_err("mk_spawn: Spawn context layout mismatch - host and spawn kernels must be built from the same source\n"); - return; - } - + stamped_phys = READ_ONCE(ctx->self_phys); + if (stamped_phys != ctx_phys) + mk_reject_spawn_context(); + abi_magic = READ_ONCE(ctx->abi_magic); + if (abi_magic != MK_BOOT_CONTEXT_MAGIC) + mk_reject_spawn_context(); + + mk_boot_context_phys = ctx_phys; mk_boot_context = ctx; + return ctx; +} + +/* + * Initialize boot context tracking in spawn kernel. + * Called early during spawn kernel boot. + */ +void mk_init_boot_context(phys_addr_t ctx_phys) +{ + struct mk_spawn_context *ctx = mk_validate_boot_context(ctx_phys); + + if (!ctx) + return; /* * A spawn kernel cannot calibrate against legacy timers because they * belong to the host. Reuse the selected physical CPU's delay and local @@ -748,19 +797,27 @@ void mk_init_boot_context(phys_addr_t ctx_phys) * * One physical page serves every wake path of this instance: the host * allocates it once in mk_setup_trampoline() and reuses it across - * re-spawns, and mk_prepare_trampoline() places our own trampoline copy + * re-spawns, and mk_arch_prepare_park() places our own trampoline copy * (including the secondary entry) in the same page. The copy matters: * the page arrives holding the HOST kernel's trampoline code, and when * the two binaries differ its offsets are wrong for this kernel. */ -static int __init mk_prepare_trampoline(void) +int __init mk_arch_prepare_park(void) { struct mk_spawn_context *ctx = mk_own_context(); unsigned long virt; int ret; + if (mk_arch_park_ready()) + return 0; + if (spawn_park_error) + return spawn_park_error; if (!ctx) return 0; + if (!ctx->trampoline_phys || !ctx->park_phys || !ctx->park_cr3) { + ret = -EINVAL; + goto fail; + } /* * Put our own copy of the trampoline in the page the host set @@ -768,32 +825,48 @@ static int __init mk_prepare_trampoline(void) * is entered from an offline CPU, where changing page attributes * is not allowed. */ - memcpy(__va(ctx->trampoline_phys), multikernel_relocate_kernel_start, - multikernel_relocate_kernel_end - multikernel_relocate_kernel_start); + if (!spawn_trampoline_prepared) { + spawn_trampoline_phys = ctx->trampoline_phys; + spawn_trampoline_va = __va(spawn_trampoline_phys); + memcpy(spawn_trampoline_va, multikernel_relocate_kernel_start, + multikernel_relocate_kernel_end - + multikernel_relocate_kernel_start); - /* - * Both pages are executed from the direct map, which is writable, - * so drop write before adding execute. Leaving them writable and - * executable trips the kernel's own W^X check. - */ - virt = (unsigned long)__va(ctx->trampoline_phys) & PAGE_MASK; - ret = set_memory_ro(virt, 1); - if (!ret) - ret = set_memory_x(virt, 1); - if (ret) - return ret; + /* + * Both pages are executed from the direct map, which is writable, + * so drop write before adding execute. Leaving them writable and + * executable trips the kernel's own W^X check. + */ + virt = (unsigned long)spawn_trampoline_va & PAGE_MASK; + ret = set_memory_ro(virt, 1); + if (!ret) + ret = set_memory_x(virt, 1); + if (ret) + goto fail; + spawn_trampoline_prepared = true; + } /* The pool park page is entered the same way when this kernel dies */ - if (ctx->park_phys) { + if (!spawn_pool_park_prepared) { virt = (unsigned long)__va(ctx->park_phys) & PAGE_MASK; ret = set_memory_ro(virt, 1); if (!ret) ret = set_memory_x(virt, 1); + if (ret) + goto fail; + spawn_pool_park_prepared = true; } + /* Publish executable mappings before any reject or abort can park. */ + smp_store_release(&spawn_park_ready, true); + return 0; + +fail: + /* A partial W^X transition is not safe to retry. */ + spawn_park_error = ret; return ret; } -early_initcall(mk_prepare_trampoline); +early_initcall(mk_arch_prepare_park); /* * Add a 2MB executable mapping to a page table. diff --git a/drivers/tty/mktty.c b/drivers/tty/mktty.c index 4a4d3f68eaf011..535c6c1fe8a3e7 100644 --- a/drivers/tty/mktty.c +++ b/drivers/tty/mktty.c @@ -337,6 +337,17 @@ static struct tty_driver *mktty_spawn_driver; static struct mktty_spawn_state mktty_spawn; static struct mk_ipi_handler *mktty_spawn_handler; static struct console mktty_spawn_console; + +static int mktty_spawn_parent_id(void) +{ + if (!mk_self) + return -ENODEV; + if (mk_self->id == 0) + return 0; + if (!host_instance) + return -ENODEV; + return READ_ONCE(host_instance->id); +} static bool mktty_console_registered; static int mktty_spawn_activate(struct tty_port *port, struct tty_struct *tty) @@ -376,11 +387,14 @@ static ssize_t mktty_spawn_write(struct tty_struct *tty, const u8 *buf, size_t count) { struct mktty_message *msg; - size_t sent = 0, chunk; - int ret; + size_t sent = 0, chunk, msg_size; + int ret, parent_id; if (tty->index != 0) return -ENODEV; + parent_id = mktty_spawn_parent_id(); + if (parent_id < 0) + return parent_id; msg = kmalloc(sizeof(*msg), GFP_KERNEL); if (!msg) @@ -393,10 +407,10 @@ static ssize_t mktty_spawn_write(struct tty_struct *tty, const u8 *buf, msg->len = chunk; msg->reserved = 0; memcpy(msg->data, buf + sent, chunk); + msg_size = sizeof(*msg) - MKTTY_MAX_DATA + chunk; - ret = multikernel_send_ipi_data(0, msg, - sizeof(*msg) - MKTTY_MAX_DATA + chunk, - MKTTY_IPI_TYPE); + ret = multikernel_send_ipi_data(parent_id, msg, msg_size, + MKTTY_IPI_TYPE); if (ret < 0) { kfree(msg); return sent > 0 ? sent : ret; @@ -445,12 +459,16 @@ static struct mktty_message mktty_console_msg; static DEFINE_SPINLOCK(mktty_console_lock); static void mktty_spawn_console_write(struct console *con, const char *s, - unsigned int count) + unsigned int count) { unsigned long flags; - size_t chunk; + size_t chunk, msg_size; + int parent_id; spin_lock_irqsave(&mktty_console_lock, flags); + parent_id = mktty_spawn_parent_id(); + if (parent_id < 0) + goto out; while (count > 0) { chunk = min_t(size_t, count, MKTTY_MAX_DATA); mktty_console_msg.type = MKTTY_MSG_OUTPUT; @@ -458,13 +476,14 @@ static void mktty_spawn_console_write(struct console *con, const char *s, mktty_console_msg.len = chunk; mktty_console_msg.reserved = 0; memcpy(mktty_console_msg.data, s, chunk); + msg_size = sizeof(mktty_console_msg) - MKTTY_MAX_DATA + chunk; - multikernel_send_ipi_data(0, &mktty_console_msg, - sizeof(mktty_console_msg) - MKTTY_MAX_DATA + chunk, - MKTTY_IPI_TYPE); + (void)multikernel_send_ipi_data_to_host(&mktty_console_msg, + msg_size, MKTTY_IPI_TYPE); s += chunk; count -= chunk; } +out: spin_unlock_irqrestore(&mktty_console_lock, flags); } diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index 1ec6a0d5170005..09460a911bb9cc 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -16,6 +16,11 @@ #include #include #include +#include +#include + +struct pci_bus; +struct mk_instance; #ifdef CONFIG_MULTIKERNEL #include @@ -40,6 +45,7 @@ typedef u64 mk_phys_cpu_t; #define MK_PHYS_CPU_INVALID (~(mk_phys_cpu_t)0) struct mk_cpu_set { + raw_spinlock_t lock; unsigned int nr; /* Entries in use */ unsigned int cap; /* Allocated capacity */ mk_phys_cpu_t *ids; @@ -55,25 +61,15 @@ bool mk_cpu_set_contains(const struct mk_cpu_set *set, mk_phys_cpu_t id); int mk_cpu_set_copy(struct mk_cpu_set *dst, const struct mk_cpu_set *src); int mk_cpu_set_format(char *buf, size_t size, const struct mk_cpu_set *set); -static inline unsigned int mk_cpu_set_count(const struct mk_cpu_set *set) -{ - return set ? set->nr : 0; -} - -static inline bool mk_cpu_set_empty(const struct mk_cpu_set *set) -{ - return mk_cpu_set_count(set) == 0; -} - -static inline mk_phys_cpu_t mk_cpu_set_first(const struct mk_cpu_set *set) -{ - return mk_cpu_set_empty(set) ? MK_PHYS_CPU_INVALID : set->ids[0]; -} +unsigned int mk_cpu_set_count(const struct mk_cpu_set *set); +bool mk_cpu_set_empty(const struct mk_cpu_set *set); +mk_phys_cpu_t mk_cpu_set_first(const struct mk_cpu_set *set); +bool mk_cpu_set_get(const struct mk_cpu_set *set, unsigned int index, + mk_phys_cpu_t *id); #define mk_cpu_set_for_each(i, id, set) \ for ((i) = 0; \ - (set) && (i) < (set)->nr && \ - (((id) = (set)->ids[(i)]), true); \ + mk_cpu_set_get((set), (i), &(id)); \ (i)++) /** @@ -88,16 +84,15 @@ static inline mk_phys_cpu_t mk_cpu_set_first(const struct mk_cpu_set *set) /* Data structure for passing parameters via IPI */ struct mk_ipi_data { + u32 ready; u64 sender_cpu; /* Physical ID of the CPU that sent this IPI */ - unsigned int type; /* User-defined type identifier */ + unsigned int type; /* User-defined type identifier */ size_t data_size; /* Size of the data */ char buffer[MK_MAX_DATA_SIZE]; /* Actual data buffer */ }; /* IPI ring buffer for queuing messages */ struct mk_ipi_ring { - atomic_t head; /* Producer index */ - atomic_t tail; /* Consumer index */ struct mk_ipi_data entries[MK_IPI_RING_SIZE]; /* Ring buffer entries */ }; @@ -109,9 +104,10 @@ struct mk_ipi_ring { /* Presence table capacity, in CPUs (see mk_cpu_rank) */ #define MK_PARKED_MAX 512 -/* Shared memory structures - per-instance design */ +/* One duplex link per parent/child pair. Each ring has one kernel producer. */ struct mk_shared_data { - struct mk_ipi_ring ring; /* IPI message ring buffer */ + struct mk_ipi_ring to_child; + struct mk_ipi_ring to_parent; /* * Force-halt marker, host-owned. Armed before the host NMIs the * instance's CPUs and cleared with the rest of this struct when @@ -131,6 +127,40 @@ struct mk_shared_data { * the struct on re-exec. */ u8 parked[MK_PARKED_MAX]; + s32 parent_id; + s32 child_id; + u32 reserved; + u64 parent_doorbell_cpu; + u64 child_doorbell_cpu; +}; + +static inline void mk_ipi_ring_reset(struct mk_ipi_ring *ring) +{ + unsigned int i; + + for (i = 0; i < MK_IPI_RING_SIZE; i++) + WRITE_ONCE(ring->entries[i].ready, 0); +} + +static inline void mk_shared_data_reset(struct mk_shared_data *shared) +{ + mk_ipi_ring_reset(&shared->to_child); + mk_ipi_ring_reset(&shared->to_parent); + WRITE_ONCE(shared->force_halt, 0); +} + +struct mk_ipi_endpoint { + struct mk_ipi_ring *tx; + struct mk_ipi_ring *rx; + raw_spinlock_t tx_lock; + raw_spinlock_t rx_lock; + u32 tx_head; + u32 rx_tail; + bool tx_enabled; + bool rx_dispatching; + bool parent_side; + bool registered; + struct list_head rx_node; }; /* Function pointer type for IPI callbacks */ @@ -172,11 +202,18 @@ void multikernel_unregister_handler(struct mk_ipi_handler *handler); * Returns 0 on success, negative error code on failure */ int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, unsigned long type); +int multikernel_send_ipi_data_to_host(void *data, size_t data_size, + unsigned long type); void generic_multikernel_interrupt(void); -/* Discard everything queued in this kernel's ring (instance re-spawn) */ -void mk_ipi_ring_drop_pending(void); +int mk_ipi_endpoint_init(struct mk_instance *instance, bool parent_side); +void mk_ipi_endpoint_unregister(struct mk_instance *instance); +void mk_ipi_endpoint_close(struct mk_instance *instance); +void mk_ipi_link_reset(struct mk_instance *instance, int parent_id, + int child_id, mk_phys_cpu_t parent_cpu, + mk_phys_cpu_t child_cpu); +void mk_ipi_handlers_enable(void); /* * Multikernel Messaging System @@ -655,6 +692,11 @@ struct mk_instance { struct mk_shared_data *ipi_data; /* IPI shared memory buffer (virtual address) */ phys_addr_t ipi_phys; /* IPI buffer physical address */ u32 ipi_pages; /* IPI buffer size in pages */ + /* + * Separate host self area used when a spawn fences its parent. + * NULL for ordinary child records, whose halt area is @ipi_data. + */ + struct mk_shared_data *halt_data; /* * On a spawn's record of its host: the physical address of the * host's pool wake slot, where the host's CPUs park. A backup @@ -669,6 +711,7 @@ struct mk_instance { * explicitly so routing never masquerades as ownership in @cpus. */ mk_phys_cpu_t ipi_target; + struct mk_ipi_endpoint ipi_endpoint; /* Kexec integration */ struct kimage *kimage; /* Associated kimage object */ @@ -922,6 +965,7 @@ struct mk_instance *mk_instance_find(int mk_id); void mk_instance_put(struct mk_instance *instance); void mk_instance_set_state(struct mk_instance *instance, enum mk_instance_state state); +int mk_instance_abort_spawn(struct mk_instance *instance); /* Kimage-based access to the instance memory pool */ void *mk_kimage_alloc(struct kimage *image, size_t size, size_t align); @@ -936,6 +980,7 @@ void mk_register_cpus_from_manifest(void); /* Accept the manifest handed over at boot (spawn kernels) */ void mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len); +bool mk_manifest_rejected(void); /* Build the manifest for a spawn (host, kexec path) */ int mk_manifest_finalize(struct kimage *image); @@ -991,6 +1036,11 @@ static inline void mk_register_cpus_from_manifest(void) static inline void mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) { } + +static inline bool mk_manifest_rejected(void) +{ + return false; +} #endif /** @@ -998,6 +1048,7 @@ static inline void mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) */ #define MK_DT_CONFIG_VERSION_1 1 #define MK_DT_CONFIG_CURRENT MK_DT_CONFIG_VERSION_1 +/* Bumped whenever the shared-memory layout or message semantics change. */ #define MK_FDT_COMPATIBLE "multikernel-v1" /** @@ -1015,8 +1066,6 @@ static inline void mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) * the host side, and restore an instance from it on the spawn side. */ - - /** * mk_instance_restore_from_manifest() - Restore this instance from the manifest * @@ -1099,6 +1148,8 @@ void mk_arch_register_cpu(mk_phys_cpu_t phys_id); /* Park the calling CPU in the pool wait loop; never returns */ void __noreturn mk_enter_pool_state(void *info); +int __init mk_arch_prepare_park(void); +bool mk_arch_park_ready(void); /* * Forcible stop of another instance's CPUs (NMI on x86, SDEI or diff --git a/include/linux/multikernel_abi.h b/include/linux/multikernel_abi.h new file mode 100644 index 00000000000000..3d9bb5fdeee61e --- /dev/null +++ b/include/linux/multikernel_abi.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _LINUX_MULTIKERNEL_ABI_H +#define _LINUX_MULTIKERNEL_ABI_H + +/* One generation marker shared by the spawn context and image capabilities. */ +#define MK_BOOT_CONTEXT_MAGIC 0x4d4b0002 +#define MK_VMLINUX_NOTE_TYPE 0x4d4b0002 + +#endif /* _LINUX_MULTIKERNEL_ABI_H */ diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index e1d0d55b1a0f46..709838b7dfe1da 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -609,6 +609,12 @@ void kimage_free(struct kimage *image) if (image->type == KEXEC_TYPE_MULTIKERNEL) { unsigned long i; + /* Stop delivery before image-owned shared pages are returned. */ +#ifdef CONFIG_MULTIKERNEL + if (image->mk_instance) + mk_ipi_endpoint_unregister(image->mk_instance); +#endif + for (i = 0; i < image->nr_segments; i++) { void *virt_addr = phys_to_virt(image->segment[i].mem); @@ -1701,6 +1707,12 @@ int multikernel_kexec_by_id(int mk_id) } instance = mk_image->mk_instance; + if (instance->state != MK_STATE_LOADED) { + pr_err("Multikernel instance %d is not loadable (state=%d)\n", + mk_id, instance->state); + rc = -EINVAL; + goto unlock; + } if (!mk_cpu_set_empty(instance->cpus)) { mk_phys_cpu_t phys_cpu = mk_cpu_set_first(instance->cpus); @@ -1755,10 +1767,11 @@ int multikernel_kexec_by_id(int mk_id) } rc = mk_manifest_finalize(mk_image); - if (rc) - pr_warn("Manifest finalization failed: %d\n", rc); - else - pr_info("Manifest finalized for multikernel instance\n"); + if (rc) { + pr_err("Manifest finalization failed: %d\n", rc); + goto unlock; + } + pr_info("Manifest finalized for multikernel instance\n"); /* * Point at the ring this image actually carries. Every load @@ -1775,32 +1788,36 @@ int multikernel_kexec_by_id(int mk_id) PAGE_ALIGN(sizeof(struct mk_shared_data)) >> PAGE_SHIFT; } - /* - * Start the instance with an empty ring. It outlives the kernel - * that was using it, so a new instance would otherwise inherit that - * kernel's indices and any slot it left half written - which stalls - * the reader, since an unpublished slot means "the sender is still - * filling this one". Anything left in there was addressed to a - * kernel that is gone. - */ - if (instance->ipi_data) - memset(instance->ipi_data, 0, sizeof(*instance->ipi_data)); + /* Reset only this parent/child link, after the old child is parked. */ + if (instance->ipi_data) { + mk_ipi_link_reset(instance, mk_self->id, mk_id, + mk_cpu_set_first(mk_self->cpus), + mk_cpu_set_first(instance->cpus)); + } + rc = mk_arch_spawn_instance(mk_image, instance, cpu); + if (rc) { + mk_ipi_endpoint_close(instance); + goto unlock; + } /* - * Same for the other direction: whatever the halted instance left - * queued for us is addressed from a kernel that no longer exists, - * and a slot it claimed but never published stalls our ring for - * good. + * The instance is running once its CPUs leave the park loop. Publish that + * state before dropping the global kexec lock so another exec cannot race + * this boot while the readiness handshake is pending. */ - mk_ipi_ring_drop_pending(); + rc = mk_instance_set_kexec_active(mk_image->mk_id); + if (rc) { + int abort_ret = mk_instance_abort_spawn(instance); - rc = mk_arch_spawn_instance(mk_image, instance, cpu); - if (rc == 0) { - rc = mk_instance_set_kexec_active(mk_image->mk_id); - if (rc) - pr_warn("Failed to set instance %d as active: %d\n", mk_image->mk_id, rc); + if (abort_ret) + pr_crit("Instance %d activation abort failed: %d\n", + mk_id, abort_ret); + goto unlock; } + kexec_unlock(); + return 0; + unlock: kexec_unlock(); return rc; diff --git a/kernel/multikernel/Kconfig b/kernel/multikernel/Kconfig index cec1ea32d83361..e7e242d0feac37 100644 --- a/kernel/multikernel/Kconfig +++ b/kernel/multikernel/Kconfig @@ -17,6 +17,7 @@ config MULTIKERNEL depends on KEXEC_CORE depends on MEMORY_HOTPLUG depends on MEMORY_HOTREMOVE + depends on OF select LIBFDT select OF_DYNAMIC if OF help diff --git a/kernel/multikernel/core.c b/kernel/multikernel/core.c index 9de50f787d2737..cecdebdcc36d6c 100644 --- a/kernel/multikernel/core.c +++ b/kernel/multikernel/core.c @@ -171,6 +171,7 @@ static void mk_instance_release(struct kref *kref) pr_info("Releasing multikernel instance %d (%s), returning resources to root\n", instance->id, instance->name); + mk_ipi_endpoint_unregister(instance); mk_instance_return_all_cpus(instance); mk_instance_return_pci_devices(instance); @@ -178,6 +179,8 @@ static void mk_instance_release(struct kref *kref) mk_instance_free_memory(instance); mk_instance_track_dump(instance, instance->state, MK_STATE_READY); + if (instance->halt_data) + memunmap(instance->halt_data); kfree(instance->host_tree); mk_cpu_set_free(instance->cpus); kfree(instance->name); @@ -231,6 +234,9 @@ struct mk_instance *mk_instance_alloc(int id, const char *name) instance->state = MK_STATE_READY; instance->ipi_target = MK_PHYS_CPU_INVALID; + raw_spin_lock_init(&instance->ipi_endpoint.tx_lock); + raw_spin_lock_init(&instance->ipi_endpoint.rx_lock); + INIT_LIST_HEAD(&instance->ipi_endpoint.rx_node); INIT_LIST_HEAD(&instance->memory_regions); INIT_LIST_HEAD(&instance->list); INIT_LIST_HEAD(&instance->pci_devices); @@ -297,6 +303,8 @@ void mk_instance_free(struct mk_instance *instance) list_del(&plat_dev->list); kfree(plat_dev); } + if (instance->halt_data) + memunmap(instance->halt_data); kfree(instance->host_tree); mk_cpu_set_free(instance->cpus); kfree(instance->name); @@ -423,12 +431,25 @@ bool multikernel_allow_emergency_restart(void) */ int mk_instance_confirm_parked(struct mk_instance *instance) { + struct mk_cpu_set *snapshot; mk_phys_cpu_t phys_cpu; unsigned int i; int ret, failed = 0; /* Empty until the instance first ran, so nothing of it is executing */ - mk_cpu_set_for_each(i, phys_cpu, instance->cpus_on_slot) { + if (!instance->cpus_on_slot) + return 0; + + snapshot = mk_cpu_set_alloc(); + if (!snapshot) + return -ENOMEM; + ret = mk_cpu_set_copy(snapshot, instance->cpus_on_slot); + if (ret) { + mk_cpu_set_free(snapshot); + return ret; + } + + mk_cpu_set_for_each(i, phys_cpu, snapshot) { ret = mk_arch_confirm_parked(instance, phys_cpu); if (ret) { pr_err("Instance %d (%s): CPU %llu is not parked: %d\n", @@ -436,6 +457,7 @@ int mk_instance_confirm_parked(struct mk_instance *instance) failed++; } } + mk_cpu_set_free(snapshot); return failed ? -EBUSY : 0; } @@ -1542,7 +1564,12 @@ static void __noreturn mk_notify_down_and_park(int target_id, u32 subtype) */ void __noreturn mk_halt_to_pool(void) { - mk_notify_down_and_park(0, MK_SYS_HALTED); + int parent_id; + + if (!host_instance) + panic("multikernel: spawned kernel has no parent instance"); + parent_id = READ_ONCE(host_instance->id); + mk_notify_down_and_park(parent_id, MK_SYS_HALTED); } static void mk_shutdown_work_fn(struct work_struct *work) @@ -1717,7 +1744,7 @@ int multikernel_halt_by_id(int mk_id) return ret; } -/** +/* * multikernel_force_halt_by_id - Forcible shutdown of a multikernel instance via NMI * @mk_id: Instance ID to halt * @@ -1788,7 +1815,7 @@ EXPORT_SYMBOL_GPL(mk_cpu_rank); static int mk_fence_missing(struct mk_instance *instance, struct mk_cpu_set *targets) { - struct mk_shared_data *sd = instance->ipi_data; + struct mk_shared_data *sd = mk_instance_halt_data(instance); mk_phys_cpu_t phys; unsigned int i; int missing = 0; @@ -1813,7 +1840,7 @@ static int mk_fence_missing(struct mk_instance *instance, static int mk_confirm_fenced(struct mk_instance *instance, struct mk_cpu_set *targets) { - struct mk_shared_data *sd = instance->ipi_data; + struct mk_shared_data *sd = mk_instance_halt_data(instance); mk_phys_cpu_t phys; unsigned int i; int missing, ret; @@ -1847,22 +1874,20 @@ static int mk_confirm_fenced(struct mk_instance *instance, return ret; } -int multikernel_force_halt_by_id(int mk_id) +static int __mk_instance_force_halt(struct mk_instance *instance, + bool allow_loaded) { - struct mk_instance *instance; struct mk_cpu_set *targets; mk_phys_cpu_t phys_cpu; unsigned int i; int cpu_count = 0; int ret; - instance = mk_instance_find(mk_id); if (!instance) - return -ENOENT; + return -EINVAL; if (instance == mk_self) { - pr_err("Cannot force halt this kernel (id %d)\n", mk_id); - mk_instance_put(instance); + pr_err("Cannot force halt this kernel (id %d)\n", instance->id); return -EINVAL; } @@ -1873,29 +1898,27 @@ int multikernel_force_halt_by_id(int mk_id) * a rerun the instance is stuck for good. The parent is ACTIVE. */ if (instance->state != MK_STATE_ACTIVE && - instance->state != MK_STATE_LOADED) { + (!allow_loaded || instance->state != MK_STATE_LOADED)) { pr_err("Instance %d not running (state=%d), nothing to force halt\n", - mk_id, instance->state); - mk_instance_put(instance); + instance->id, instance->state); return -EINVAL; } targets = mk_cpu_set_alloc(); - if (!targets) { - mk_instance_put(instance); + if (!targets) return -ENOMEM; - } ret = mk_force_halt_targets(instance, targets); if (!ret && mk_cpu_set_empty(targets)) ret = -EINVAL; if (ret) { - pr_err("Instance %d: no force-halt targets: %d\n", mk_id, ret); + pr_err("Instance %d: no force-halt targets: %d\n", + instance->id, ret); mk_cpu_set_free(targets); - mk_instance_put(instance); return ret; } - pr_info("Force halting multikernel instance %d via NMI\n", mk_id); + pr_info("Force halting multikernel instance %d via NMI\n", + instance->id); ret = mk_arm_force_halt(instance); if (ret) @@ -1906,7 +1929,8 @@ int multikernel_force_halt_by_id(int mk_id) cpu_count++; } - pr_info("Sent NMI to %d CPUs in instance %d\n", cpu_count, mk_id); + pr_info("Sent NMI to %d CPUs in instance %d\n", + cpu_count, instance->id); /* * A child instance parks on its own context, so wait for it to @@ -1918,12 +1942,59 @@ int multikernel_force_halt_by_id(int mk_id) * fence, so a baseline cannot claim a machine with a CPU still * running the dead host. */ - if (instance == host_instance) + if (instance == host_instance) { ret = mk_confirm_fenced(instance, targets); - else - mk_instance_settle_halted(instance); + } else { + ret = mk_instance_confirm_parked(instance); + if (ret) + pr_err("Instance %d CPUs did not park after force halt: %d\n", + instance->id, ret); + else + mk_instance_settle_halted(instance); + } mk_cpu_set_free(targets); + return ret; +} + +int mk_instance_abort_spawn(struct mk_instance *instance) +{ + int ret; + + mk_ipi_endpoint_close(instance); + ret = __mk_instance_force_halt(instance, true); + if (ret && instance) + mk_instance_set_state(instance, MK_STATE_FAILED); + return ret; +} + +/** + * mk_instance_force_halt - Forcibly stop an instance via NMI + * @instance: Instance to stop + * + * Forces a spawn kernel's CPUs to stop by arming the persistent force-halt + * marker and sending NMIs directly to each CPU. The NMI handler checks the + * marker and parks the CPU if it is set. + * + * Use when: The spawn kernel is stuck/crashed and not responding to graceful + * shutdown, or when graceful shutdown has failed. + * + * Returns: 0 on success, negative error code on failure + */ +int mk_instance_force_halt(struct mk_instance *instance) +{ + return __mk_instance_force_halt(instance, false); +} + +int multikernel_force_halt_by_id(int mk_id) +{ + struct mk_instance *instance; + int ret; + + instance = mk_instance_find(mk_id); + if (!instance) + return -ENOENT; + ret = mk_instance_force_halt(instance); mk_instance_put(instance); return ret; } @@ -1962,6 +2033,8 @@ static int __init multikernel_init(void) return ret; } + mk_ipi_handlers_enable(); + pr_info("Multikernel support initialized\n"); return 0; } diff --git a/kernel/multikernel/cpuset.c b/kernel/multikernel/cpuset.c index ad36a4f94fc65a..f3a1a918fe4573 100644 --- a/kernel/multikernel/cpuset.c +++ b/kernel/multikernel/cpuset.c @@ -13,31 +13,22 @@ #include #include -struct mk_cpu_set *mk_cpu_set_alloc(void) -{ - return kzalloc(sizeof(struct mk_cpu_set), GFP_KERNEL); -} - -void mk_cpu_set_free(struct mk_cpu_set *set) +static void mk_cpu_set_lock(const struct mk_cpu_set *set, unsigned long *flags) { - if (!set) - return; - - kfree(set->ids); - kfree(set); + raw_spin_lock_irqsave((raw_spinlock_t *)&set->lock, *flags); } -void mk_cpu_set_clear(struct mk_cpu_set *set) +static void mk_cpu_set_unlock(const struct mk_cpu_set *set, unsigned long flags) { - if (set) - set->nr = 0; + raw_spin_unlock_irqrestore((raw_spinlock_t *)&set->lock, flags); } -static int mk_cpu_set_index(const struct mk_cpu_set *set, mk_phys_cpu_t id) +static int mk_cpu_set_index_locked(const struct mk_cpu_set *set, + mk_phys_cpu_t id) { unsigned int i; - for (i = 0; set && i < set->nr; i++) { + for (i = 0; i < set->nr; i++) { if (set->ids[i] == id) return i; } @@ -45,9 +36,35 @@ static int mk_cpu_set_index(const struct mk_cpu_set *set, mk_phys_cpu_t id) return -1; } -bool mk_cpu_set_contains(const struct mk_cpu_set *set, mk_phys_cpu_t id) +struct mk_cpu_set *mk_cpu_set_alloc(void) +{ + struct mk_cpu_set *set; + + set = kzalloc_obj(*set, GFP_KERNEL); + if (set) + raw_spin_lock_init(&set->lock); + return set; +} + +void mk_cpu_set_free(struct mk_cpu_set *set) +{ + if (!set) + return; + + kfree(set->ids); + kfree(set); +} + +void mk_cpu_set_clear(struct mk_cpu_set *set) { - return mk_cpu_set_index(set, id) >= 0; + unsigned long flags; + + if (!set) + return; + + mk_cpu_set_lock(set, &flags); + set->nr = 0; + mk_cpu_set_unlock(set, flags); } /** @@ -61,64 +78,209 @@ bool mk_cpu_set_contains(const struct mk_cpu_set *set, mk_phys_cpu_t id) */ int mk_cpu_set_reserve(struct mk_cpu_set *set, unsigned int extra) { - unsigned int cap = set->nr + extra; - mk_phys_cpu_t *ids; + mk_phys_cpu_t *ids = NULL; + mk_phys_cpu_t *old_ids; + unsigned int cap; + unsigned long flags; - if (cap <= set->cap) - return 0; + if (!set) + return -EINVAL; + + for (;;) { + mk_cpu_set_lock(set, &flags); + cap = set->nr + extra; + if (cap <= set->cap) { + mk_cpu_set_unlock(set, flags); + kfree(ids); + return 0; + } + mk_cpu_set_unlock(set, flags); - cap = max_t(unsigned int, cap, 8); - ids = krealloc_array(set->ids, cap, sizeof(*ids), GFP_KERNEL); - if (!ids) - return -ENOMEM; + cap = max_t(unsigned int, cap, 8); + kfree(ids); + ids = kcalloc(cap, sizeof(*ids), GFP_KERNEL); + if (!ids) + return -ENOMEM; - set->ids = ids; - set->cap = cap; - return 0; + mk_cpu_set_lock(set, &flags); + if (set->nr + extra > cap) { + mk_cpu_set_unlock(set, flags); + continue; + } + if (cap <= set->cap) { + mk_cpu_set_unlock(set, flags); + kfree(ids); + return 0; + } + + memcpy(ids, set->ids, set->nr * sizeof(*ids)); + old_ids = set->ids; + set->ids = ids; + set->cap = cap; + mk_cpu_set_unlock(set, flags); + kfree(old_ids); + return 0; + } } /* Idempotent: adding an ID already in the set succeeds without effect */ int mk_cpu_set_add(struct mk_cpu_set *set, mk_phys_cpu_t id) { + unsigned long flags; int ret; - if (mk_cpu_set_contains(set, id)) - return 0; + if (!set) + return -EINVAL; - ret = mk_cpu_set_reserve(set, 1); - if (ret) - return ret; + for (;;) { + mk_cpu_set_lock(set, &flags); + if (mk_cpu_set_index_locked(set, id) >= 0) { + mk_cpu_set_unlock(set, flags); + return 0; + } + if (set->nr < set->cap) { + set->ids[set->nr++] = id; + mk_cpu_set_unlock(set, flags); + return 0; + } + mk_cpu_set_unlock(set, flags); - set->ids[set->nr++] = id; - return 0; + ret = mk_cpu_set_reserve(set, 1); + if (ret) + return ret; + } } bool mk_cpu_set_del(struct mk_cpu_set *set, mk_phys_cpu_t id) { - int idx = mk_cpu_set_index(set, id); + unsigned long flags; + int idx; + + if (!set) + return false; - if (idx < 0) + mk_cpu_set_lock(set, &flags); + idx = mk_cpu_set_index_locked(set, id); + if (idx < 0) { + mk_cpu_set_unlock(set, flags); return false; + } memmove(&set->ids[idx], &set->ids[idx + 1], (set->nr - idx - 1) * sizeof(set->ids[0])); set->nr--; + mk_cpu_set_unlock(set, flags); return true; } +bool mk_cpu_set_contains(const struct mk_cpu_set *set, mk_phys_cpu_t id) +{ + unsigned long flags; + bool found; + + if (!set) + return false; + + mk_cpu_set_lock(set, &flags); + found = mk_cpu_set_index_locked(set, id) >= 0; + mk_cpu_set_unlock(set, flags); + return found; +} + +unsigned int mk_cpu_set_count(const struct mk_cpu_set *set) +{ + unsigned long flags; + unsigned int nr; + + if (!set) + return 0; + + mk_cpu_set_lock(set, &flags); + nr = set->nr; + mk_cpu_set_unlock(set, flags); + return nr; +} + +bool mk_cpu_set_empty(const struct mk_cpu_set *set) +{ + return mk_cpu_set_count(set) == 0; +} + +mk_phys_cpu_t mk_cpu_set_first(const struct mk_cpu_set *set) +{ + unsigned long flags; + mk_phys_cpu_t id; + + if (!set) + return MK_PHYS_CPU_INVALID; + + mk_cpu_set_lock(set, &flags); + id = set->nr ? set->ids[0] : MK_PHYS_CPU_INVALID; + mk_cpu_set_unlock(set, flags); + return id; +} + +bool mk_cpu_set_get(const struct mk_cpu_set *set, unsigned int index, + mk_phys_cpu_t *id) +{ + unsigned long flags; + bool found = false; + + if (!set || !id) + return false; + + mk_cpu_set_lock(set, &flags); + if (index < set->nr) { + *id = set->ids[index]; + found = true; + } + mk_cpu_set_unlock(set, flags); + return found; +} + int mk_cpu_set_copy(struct mk_cpu_set *dst, const struct mk_cpu_set *src) { - unsigned int nr = mk_cpu_set_count(src); + unsigned long src_flags; + unsigned long dst_flags; + mk_phys_cpu_t *ids; + unsigned int nr; int ret; - dst->nr = 0; - ret = mk_cpu_set_reserve(dst, nr); - if (ret) - return ret; + if (!dst || !src) + return -EINVAL; + if (dst == src) + return 0; - memcpy(dst->ids, src->ids, nr * sizeof(dst->ids[0])); - dst->nr = nr; - return 0; + for (;;) { + nr = mk_cpu_set_count(src); + ids = nr ? kmalloc_array(nr, sizeof(*ids), GFP_KERNEL) : NULL; + if (nr && !ids) + return -ENOMEM; + + mk_cpu_set_lock(src, &src_flags); + if (src->nr > nr) { + mk_cpu_set_unlock(src, src_flags); + kfree(ids); + continue; + } + nr = src->nr; + if (nr) + memcpy(ids, src->ids, nr * sizeof(*ids)); + mk_cpu_set_unlock(src, src_flags); + + ret = mk_cpu_set_reserve(dst, nr); + if (ret) { + kfree(ids); + return ret; + } + mk_cpu_set_lock(dst, &dst_flags); + if (nr) + memcpy(dst->ids, ids, nr * sizeof(dst->ids[0])); + dst->nr = nr; + mk_cpu_set_unlock(dst, dst_flags); + kfree(ids); + return 0; + } } /** @@ -127,22 +289,31 @@ int mk_cpu_set_copy(struct mk_cpu_set *dst, const struct mk_cpu_set *src) * @size: Buffer size * @set: Set to format * - * Writes "none" for an empty set, a comma-separated list of physical + * Writes none for an empty set, a comma-separated list of physical * IDs otherwise. Output is truncated to @size. Returns the number of * characters written. */ int mk_cpu_set_format(char *buf, size_t size, const struct mk_cpu_set *set) { + unsigned long flags; unsigned int i; int len = 0; - if (mk_cpu_set_empty(set)) + if (!buf || !size) + return 0; + if (!set) return scnprintf(buf, size, "none"); - for (i = 0; i < set->nr; i++) { + mk_cpu_set_lock(set, &flags); + if (!set->nr) { + mk_cpu_set_unlock(set, flags); + return scnprintf(buf, size, "none"); + } + + for (i = 0; i < set->nr && len < size; i++) { len += scnprintf(buf + len, size - len, "%s%llu", i ? "," : "", set->ids[i]); } - + mk_cpu_set_unlock(set, flags); return len; } diff --git a/kernel/multikernel/hotplug.c b/kernel/multikernel/hotplug.c index 91ff72e0a168d6..1e5fdf743fd374 100644 --- a/kernel/multikernel/hotplug.c +++ b/kernel/multikernel/hotplug.c @@ -633,6 +633,7 @@ static int mk_handle_mem_remove(struct mk_mem_resource_payload *payload, u32 pay * PCI Device Hotplug Operations */ +#if IS_ENABLED(CONFIG_PCI) static int mk_do_device_add(u16 domain, u8 bus, u8 devfn, const char *driver_override, u32 flags) { @@ -802,6 +803,19 @@ static int mk_do_device_remove(u16 domain, u8 bus, u8 devfn) return 0; } +#else /* CONFIG_PCI */ +static int mk_do_device_add(u16 domain, u8 bus, u8 devfn, + const char *driver_override, u32 flags) +{ + return -ENODEV; +} + +static int mk_do_device_remove(u16 domain, u8 bus, u8 devfn) +{ + return -ENODEV; +} +#endif /* CONFIG_PCI */ + struct mk_device_hotplug_work { struct work_struct work; u16 domain; diff --git a/kernel/multikernel/instance_dt.c b/kernel/multikernel/instance_dt.c index c0f6ca709c0298..cd4d437ed84137 100644 --- a/kernel/multikernel/instance_dt.c +++ b/kernel/multikernel/instance_dt.c @@ -21,6 +21,7 @@ #include #include #include +#include #include "internal.h" #define PROP_SUB_FDT "fdt" @@ -42,6 +43,19 @@ EXPORT_SYMBOL_GPL(mk_self); struct mk_instance *host_instance; EXPORT_SYMBOL_GPL(host_instance); +static void __init __noreturn mk_manifest_reject_and_park(int error) +{ + int ret; + + ret = mk_arch_prepare_park(); + if (ret || !mk_arch_park_ready()) + panic("multikernel: rejected manifest before park path became ready"); + pr_emerg("multikernel: parking CPUs after rejecting supplied manifest: %d\n", + error); + smp_call_function(mk_enter_pool_state, NULL, 0); + mk_enter_pool_state(NULL); +} + /** * mk_dt_extract_instance_info() - Extract instance ID and name from DTB * @dtb_data: Device tree blob data @@ -211,6 +225,7 @@ static int __init mk_instance_alloc_ipi(struct mk_instance *instance) instance->id); return -ENOMEM; } + mk_shared_data_reset(instance->ipi_data); instance->ipi_phys = virt_to_phys(instance->ipi_data); instance->ipi_pages = (sizeof(struct mk_shared_data) + PAGE_SIZE - 1) / PAGE_SIZE; @@ -297,6 +312,11 @@ static int __init mk_restore_instance_ipi(struct mk_instance *instance) return 0; } ipi_size = (size_t)ipi_pages << PAGE_SHIFT; + if (ipi_size < sizeof(struct mk_shared_data)) { + pr_err("IPI buffer is too small: %zu < %zu\n", ipi_size, + sizeof(struct mk_shared_data)); + return -EPROTO; + } instance->ipi_data = memremap(ipi_phys, ipi_size, MEMREMAP_WB); if (!instance->ipi_data) { @@ -316,38 +336,67 @@ static int __init mk_restore_instance_ipi(struct mk_instance *instance) static int __init mk_restore_host_instance(void) { struct mk_instance *hi; - phys_addr_t host_ipi_phys; - u32 host_ipi_pages; - size_t host_ipi_size; + struct mk_shared_data *shared; + mk_phys_cpu_t parent_cpu; + phys_addr_t halt_phys; + size_t halt_size; + int parent_id; + u32 halt_pages; + int ret; - if (!mk_chosen_ring("host-ipi", &host_ipi_phys, &host_ipi_pages)) { - pr_warn("No host IPI buffer in the boot tree (spawn won't be able to send to host)\n"); + if (!mk_self || !mk_self->ipi_data) { + pr_err("No parent/child IPI link in the boot tree\n"); return -ENOENT; } - host_ipi_size = (size_t)host_ipi_pages << PAGE_SHIFT; - - hi = mk_instance_alloc(0, "host"); + shared = mk_self->ipi_data; + parent_id = READ_ONCE(shared->parent_id); + parent_cpu = READ_ONCE(shared->parent_doorbell_cpu); + if (parent_id < 0 || parent_id == mk_self->id || + parent_cpu == MK_PHYS_CPU_INVALID) + return -EPROTO; + + hi = mk_instance_alloc(parent_id, "host"); if (!hi) return -ENOMEM; - - /* - * The host's owned CPU set is unknown here; ring its doorbell on - * physical CPU 0 without pretending we know what it owns. - */ - hi->ipi_target = 0; - - hi->ipi_data = memremap(host_ipi_phys, host_ipi_size, MEMREMAP_WB); - if (!hi->ipi_data) { - pr_err("Failed to map host IPI buffer at 0x%llx\n", - (unsigned long long)host_ipi_phys); + hi->ipi_target = parent_cpu; + ret = mk_cpu_set_add(hi->cpus, parent_cpu); + if (ret) + goto err_free; + if (!mk_chosen_ring("host-ipi", &halt_phys, &halt_pages)) { + pr_err("No host force-halt area in the boot tree\n"); + ret = -EPROTO; + goto err_free; + } + halt_size = (size_t)halt_pages << PAGE_SHIFT; + if (halt_size < sizeof(struct mk_shared_data) || + halt_phys == mk_self->ipi_phys) { + pr_err("Invalid host force-halt area: phys=0x%llx, pages=%u\n", + (unsigned long long)halt_phys, halt_pages); + ret = -EPROTO; goto err_free; } - hi->ipi_phys = host_ipi_phys; - hi->ipi_pages = host_ipi_pages; + hi->halt_data = memremap(halt_phys, halt_size, MEMREMAP_WB); + if (!hi->halt_data) { + pr_err("Failed to map host force-halt area at 0x%llx\n", + (unsigned long long)halt_phys); + ret = -ENOMEM; + goto err_free; + } + hi->ipi_data = mk_self->ipi_data; + hi->ipi_phys = mk_self->ipi_phys; + hi->ipi_pages = mk_self->ipi_pages; - if (mk_instance_publish(hi)) { - memunmap(hi->ipi_data); + ret = mk_instance_publish(hi); + if (ret) goto err_free; + ret = mk_ipi_endpoint_init(hi, false); + if (ret) { + mutex_lock(&mk_instance_mutex); + idr_remove(&mk_instance_idr, hi->id); + list_del(&hi->list); + mutex_unlock(&mk_instance_mutex); + mk_instance_free(hi); + return ret; } /* The host is running, or this kernel would not be */ mk_instance_set_state(hi, MK_STATE_ACTIVE); @@ -363,15 +412,13 @@ static int __init mk_restore_host_instance(void) host_instance = hi; - pr_info("Restored host IPI buffer: phys=0x%llx, pages=%u\n", - (unsigned long long)host_ipi_phys, host_ipi_pages); - pr_info("Registered host instance (ID 0) for spawn→host communication\n"); + pr_info("Registered parent instance %d on duplex IPI link\n", parent_id); return 0; err_free: mk_instance_free(hi); - return -ENOMEM; + return ret; } /** @@ -394,6 +441,9 @@ int __init mk_instance_restore_from_manifest(void) int instance_id; const char *instance_name; + if (mk_manifest_rejected()) + mk_manifest_reject_and_park(-EPROTO); + if (!mk_manifest_phys()) { pr_info("No manifest available for multikernel DTB restoration\n"); @@ -436,7 +486,7 @@ int __init mk_instance_restore_from_manifest(void) dtb_virt = initial_boot_params; if (!dtb_virt || !of_have_populated_dt()) { pr_err("Boot device tree from the manifest was not unflattened\n"); - return -ENOENT; + mk_manifest_reject_and_park(-ENOENT); } dtb_len = fdt_totalsize(dtb_virt); @@ -445,7 +495,7 @@ int __init mk_instance_restore_from_manifest(void) ret = mk_dt_extract_instance_info(dtb_virt, dtb_len, &instance_id, &instance_name); if (ret) { pr_err("Failed to extract instance info from DTB: %d\n", ret); - return ret; + mk_manifest_reject_and_park(ret); } pr_info("DTB contains instance ID %d, name '%s'\n", instance_id, instance_name); @@ -494,26 +544,33 @@ int __init mk_instance_restore_from_manifest(void) } ret = mk_instance_publish(instance); - if (ret) { - if (instance->ipi_data) - memunmap(instance->ipi_data); + if (ret) goto cleanup_instance; - } mk_self = instance; - if (mk_restore_host_instance()) - pr_warn("Failed to restore host instance (spawn→host communication unavailable)\n"); + ret = mk_restore_host_instance(); + if (ret) + mk_manifest_reject_and_park(ret); + ret = mk_arch_prepare_park(); + if (ret) + mk_manifest_reject_and_park(ret); + if (!mk_arch_park_ready()) + mk_manifest_reject_and_park(-EIO); pr_info("Successfully restored multikernel self instance %d ('%s') from the boot tree (%d bytes)\n", instance_id, instance_name, dtb_len); mk_dt_config_free(&config); return 0; cleanup_instance: + if (instance->ipi_data) + memunmap(instance->ipi_data); mk_instance_free(instance); config_free: mk_dt_config_free(&config); + if (ret) + mk_manifest_reject_and_park(ret); return ret; } @@ -533,6 +590,7 @@ early_initcall(mk_instance_restore_from_manifest); * * Returns: true if probing should proceed, false to skip entirely */ +#if IS_ENABLED(CONFIG_PCI) bool mk_pci_should_probe(struct pci_bus *bus, int devfn) { struct device_node *np; @@ -550,6 +608,7 @@ bool mk_pci_should_probe(struct pci_bus *bus, int devfn) return available; } EXPORT_SYMBOL_GPL(mk_pci_should_probe); +#endif /* CONFIG_PCI */ /* * A netdev's alias is its interface name, the one the device had in the diff --git a/kernel/multikernel/internal.h b/kernel/multikernel/internal.h index 563bad63aa0f7f..fa1c015b936e3b 100644 --- a/kernel/multikernel/internal.h +++ b/kernel/multikernel/internal.h @@ -16,6 +16,15 @@ struct mk_instance *mk_instance_alloc(int id, const char *name); int mk_instance_publish(struct mk_instance *instance); void mk_instance_free(struct mk_instance *instance); +/* core.c */ +int mk_instance_force_halt(struct mk_instance *instance); + +/* ipi.c */ +int mk_send_ipi_data(struct mk_instance *instance, void *data, + size_t data_size, unsigned long type); +struct mk_shared_data *mk_instance_halt_data(struct mk_instance *instance); +void mk_poll_ipi_messages(void); + /* kernfs.c */ extern struct kernfs_node *mk_root_kn; extern struct kernfs_node *mk_instances_kn; diff --git a/kernel/multikernel/ipi.c b/kernel/multikernel/ipi.c index b9a2a74bbf1ca4..b05fffe6019738 100644 --- a/kernel/multikernel/ipi.c +++ b/kernel/multikernel/ipi.c @@ -1,356 +1,330 @@ // SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2025 Multikernel Technologies, Inc. All rights reserved - */ -#include -#include #include -#include -#include -#include -#include -#include #include -#include +#include +#include +#include +#include +#include +#include #include "internal.h" -/* Callback management */ static struct mk_ipi_handler *mk_handlers; -static raw_spinlock_t mk_handlers_lock = __RAW_SPIN_LOCK_UNLOCKED(mk_handlers_lock); +static DEFINE_RAW_SPINLOCK(mk_handlers_lock); +static LIST_HEAD(mk_ipi_endpoints); +static DEFINE_RAW_SPINLOCK(mk_ipi_endpoints_lock); +static bool mk_handlers_ready; +static DEFINE_RATELIMIT_STATE(mk_ipi_full_rs, DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); -static void mk_ipi_drain_ring(void); +static struct mk_shared_data *mk_instance_ipi_area(struct mk_instance *instance) +{ + return READ_ONCE(instance->ipi_data); +} -/* - * Ring indices live in memory another kernel instance can write, so every - * read is masked before it indexes the entry array. An instance that dies - * mid-update must not be able to walk this kernel off the end of its ring. - */ -static inline unsigned int mk_ring_idx(unsigned int i) +struct mk_shared_data *mk_instance_halt_data(struct mk_instance *instance) { - return i & (MK_IPI_RING_SIZE - 1); + struct mk_shared_data *shared = READ_ONCE(instance->halt_data); + + return shared ? shared : mk_instance_ipi_area(instance); } -/** - * mk_ipi_ring_drop_pending - Discard everything queued in this kernel's ring - * - * Called when an instance is re-spawned. A halting instance parks its CPUs - * wherever they were, including between claiming a ring slot and publishing - * it, and the drain stops at such a slot forever. Anything still queued was - * sent by a kernel that is gone, so drop it all rather than let one - * abandoned slot wedge the ring. - */ -void mk_ipi_ring_drop_pending(void) +int mk_ipi_endpoint_init(struct mk_instance *instance, bool parent_side) +{ + struct mk_ipi_endpoint *endpoint = &instance->ipi_endpoint; + unsigned long flags; + + if (!mk_instance_ipi_area(instance)) + return -ENODEV; + if (endpoint->registered) + return 0; + endpoint->tx = parent_side ? &instance->ipi_data->to_child : + &instance->ipi_data->to_parent; + endpoint->rx = parent_side ? &instance->ipi_data->to_parent : + &instance->ipi_data->to_child; + endpoint->tx_head = 0; + endpoint->rx_tail = 0; + endpoint->tx_enabled = true; + endpoint->rx_dispatching = false; + endpoint->parent_side = parent_side; + raw_spin_lock_irqsave(&mk_ipi_endpoints_lock, flags); + list_add_tail_rcu(&endpoint->rx_node, &mk_ipi_endpoints); + endpoint->registered = true; + raw_spin_unlock_irqrestore(&mk_ipi_endpoints_lock, flags); + return 0; +} + +void mk_ipi_endpoint_close(struct mk_instance *instance) { - struct mk_ipi_ring *ring; - unsigned int head, tail; + struct mk_ipi_endpoint *endpoint = &instance->ipi_endpoint; + unsigned long flags; - if (!mk_self || !mk_self->ipi_data) + if (!endpoint->registered) return; + raw_spin_lock_irqsave(&endpoint->tx_lock, flags); + endpoint->tx_enabled = false; + raw_spin_unlock_irqrestore(&endpoint->tx_lock, flags); +} - ring = &mk_self->ipi_data->ring; - head = mk_ring_idx(atomic_read(&ring->head)); +void mk_ipi_endpoint_unregister(struct mk_instance *instance) +{ + struct mk_ipi_endpoint *endpoint = &instance->ipi_endpoint; + unsigned long flags; - for (tail = mk_ring_idx(atomic_read(&ring->tail)); tail != head; - tail = mk_ring_idx(tail + 1)) - ring->entries[tail].data_size = 0; + if (!endpoint->registered) + return; + mk_ipi_endpoint_close(instance); + raw_spin_lock_irqsave(&mk_ipi_endpoints_lock, flags); + if (endpoint->registered) { + list_del_rcu(&endpoint->rx_node); + endpoint->registered = false; + } + raw_spin_unlock_irqrestore(&mk_ipi_endpoints_lock, flags); + synchronize_rcu(); +} - atomic_set(&ring->tail, head); +void mk_ipi_link_reset(struct mk_instance *instance, int parent_id, + int child_id, mk_phys_cpu_t parent_cpu, + mk_phys_cpu_t child_cpu) +{ + struct mk_shared_data *shared = mk_instance_ipi_area(instance); + + if (!shared) + return; + mk_ipi_endpoint_unregister(instance); + mk_shared_data_reset(shared); + WRITE_ONCE(shared->parent_id, parent_id); + WRITE_ONCE(shared->child_id, child_id); + WRITE_ONCE(shared->parent_doorbell_cpu, parent_cpu); + WRITE_ONCE(shared->child_doorbell_cpu, child_cpu); + mk_ipi_endpoint_init(instance, true); } -/** - * multikernel_register_handler - Register a callback for multikernel IPI - * @callback: Function to call when IPI is received - * @ctx: Context pointer passed to the callback - * @ipi_type: IPI type this handler should process - * - * Returns pointer to handler on success, NULL on failure - */ -struct mk_ipi_handler *multikernel_register_handler(mk_ipi_callback_t callback, void *ctx, unsigned int ipi_type) +struct mk_ipi_handler * +multikernel_register_handler(mk_ipi_callback_t callback, void *ctx, + unsigned int ipi_type) { struct mk_ipi_handler *handler; unsigned long flags; if (!callback) return NULL; - handler = kzalloc(sizeof(*handler), GFP_KERNEL); if (!handler) return NULL; - handler->callback = callback; handler->context = ctx; handler->ipi_type = ipi_type; - raw_spin_lock_irqsave(&mk_handlers_lock, flags); handler->next = mk_handlers; mk_handlers = handler; raw_spin_unlock_irqrestore(&mk_handlers_lock, flags); - return handler; } EXPORT_SYMBOL(multikernel_register_handler); -/** - * multikernel_unregister_handler - Unregister a multikernel IPI callback - * @handler: Handler pointer returned from multikernel_register_handler - */ void multikernel_unregister_handler(struct mk_ipi_handler *handler) { - struct mk_ipi_handler **pp, *p; + struct mk_ipi_handler **pp, *p = NULL; unsigned long flags; if (!handler) return; - raw_spin_lock_irqsave(&mk_handlers_lock, flags); - pp = &mk_handlers; - while ((p = *pp) != NULL) { + for (pp = &mk_handlers; (p = *pp); pp = &p->next) { if (p == handler) { *pp = p->next; break; } - pp = &p->next; } raw_spin_unlock_irqrestore(&mk_handlers_lock, flags); - kfree(p); } EXPORT_SYMBOL(multikernel_unregister_handler); -/* - * An instance's IPI area is allocated when its image is loaded; the - * instance pointer is filled in lazily on first use. - */ -static struct mk_shared_data *mk_instance_ipi_area(struct mk_instance *instance) -{ - struct mk_shared_data *ipi_data; - - if (instance->ipi_data) - return instance->ipi_data; - - if (!instance->kimage || !instance->kimage->mk_ipi) - return NULL; - - ipi_data = phys_to_virt(instance->kimage->mk_ipi); - if (cmpxchg(&instance->ipi_data, NULL, ipi_data) == NULL) - pr_info("Initialized IPI ring buffer for instance %d: phys=0x%llx\n", - instance->id, (unsigned long long)instance->kimage->mk_ipi); - - return instance->ipi_data; -} - -/** - * mk_arm_force_halt - Post the force-halt marker for an instance - * @instance: Instance about to be NMIed - * - * The instance's CPUs test the marker from their NMI handlers, so it - * must be armed before the NMIs are sent. It stays armed until the - * kexec path has confirmed every CPU parked and wipes the shared area - * for the next run, which is what makes the NMI rescue idempotent: a - * repeat force halt still reaches CPUs an earlier one missed. - * - * Returns 0 on success, -ENODEV if the instance has no shared IPI area. - */ int mk_arm_force_halt(struct mk_instance *instance) { - struct mk_shared_data *ipi_data = mk_instance_ipi_area(instance); + struct mk_shared_data *shared = mk_instance_halt_data(instance); - if (!ipi_data) + if (!shared) return -ENODEV; - - WRITE_ONCE(ipi_data->force_halt, 1); - /* The marker must be visible before the NMIs that test it */ + WRITE_ONCE(shared->force_halt, 1); smp_wmb(); return 0; } -/** - * multikernel_send_ipi_data - Send data to another CPU via IPI - * @instance_id: Target multikernel instance ID - * @data: Pointer to data to send - * @data_size: Size of data - * @type: User-defined type identifier - * - * This function enqueues data into the target instance's IPI ring buffer - * and sends an IPI to notify the target CPU. - * - * Returns 0 on success, negative error code on failure - */ -int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, unsigned long type) +int mk_send_ipi_data(struct mk_instance *instance, void *data, + size_t data_size, unsigned long type) { + struct mk_ipi_endpoint *endpoint; struct mk_ipi_data *slot; - struct mk_instance *instance = mk_instance_find(instance_id); - unsigned int head, next_head, tail; + struct mk_shared_data *shared; mk_phys_cpu_t target; + unsigned long flags; + u32 idx; + int ret = 0; - if (!instance) - return -EINVAL; - if (data_size > MK_MAX_DATA_SIZE) { - mk_instance_put(instance); + if (!instance || data_size > MK_MAX_DATA_SIZE || (data_size && !data)) return -EINVAL; + endpoint = &instance->ipi_endpoint; + if (!READ_ONCE(endpoint->registered)) + return -ESHUTDOWN; + raw_spin_lock_irqsave(&endpoint->tx_lock, flags); + if (!READ_ONCE(endpoint->registered) || !endpoint->tx_enabled) { + ret = -ESHUTDOWN; + goto unlock; } - - target = instance->ipi_target; - if (target == MK_PHYS_CPU_INVALID) - target = mk_cpu_set_first(instance->cpus); + shared = READ_ONCE(instance->ipi_data); + if (!shared) { + ret = -ENODEV; + goto unlock; + } + target = endpoint->parent_side ? mk_cpu_set_first(instance->cpus) : + READ_ONCE(shared->parent_doorbell_cpu); if (target == MK_PHYS_CPU_INVALID) { - pr_err("Instance %d has no CPU to receive the IPI\n", instance_id); - mk_instance_put(instance); - return -ENODEV; + ret = -ENODEV; + goto unlock; } - - if (!mk_instance_ipi_area(instance)) { - pr_err("Multikernel IPI buffer not available for instance %d\n", instance_id); - mk_instance_put(instance); - return -ENODEV; + if (endpoint->parent_side) + WRITE_ONCE(shared->child_doorbell_cpu, target); + idx = endpoint->tx_head & (MK_IPI_RING_SIZE - 1); + slot = &endpoint->tx->entries[idx]; + /* Pair with the receiver's release when it makes the slot reusable. */ + if (smp_load_acquire(&slot->ready)) { + ret = -ENOSPC; + goto unlock; } - - /* Try to enqueue the message in the ring buffer */ - do { - head = mk_ring_idx(atomic_read(&instance->ipi_data->ring.head)); - next_head = mk_ring_idx(head + 1); - tail = mk_ring_idx(atomic_read(&instance->ipi_data->ring.tail)); - - /* Check if ring buffer is full */ - if (next_head == tail) { - /* - * Console output reaches this path, so a plain printk - * here re-enters the console write that called us and - * deadlocks on its lock with interrupts already off. - */ - printk_deferred(KERN_WARNING - "multikernel: IPI ring full for instance %d (head=%u, tail=%u)\n", - instance_id, head, tail); - mk_instance_put(instance); - return -ENOSPC; - } - - /* Try to claim this slot atomically */ - } while (atomic_cmpxchg(&instance->ipi_data->ring.head, head, next_head) != head); - - /* We've claimed slot 'head', now fill it */ - slot = &instance->ipi_data->ring.entries[head]; - - slot->sender_cpu = arch_cpu_physical_id(smp_processor_id()); - slot->type = type; - - if (data && data_size > 0) + WRITE_ONCE(slot->sender_cpu, arch_cpu_physical_id(smp_processor_id())); + WRITE_ONCE(slot->type, type); + WRITE_ONCE(slot->data_size, data_size); + if (data_size) memcpy(slot->buffer, data, data_size); + /* Publish all message fields before the receiver observes readiness. */ + smp_store_release(&slot->ready, 1); + endpoint->tx_head++; +unlock: + raw_spin_unlock_irqrestore(&endpoint->tx_lock, flags); + if (!ret) + mk_arch_send_ipi(target); + else if (ret == -ENOSPC && __ratelimit(&mk_ipi_full_rs)) + printk_deferred(KERN_WARNING + "multikernel: IPI ring full for instance %d\n", + instance->id); + return ret; +} - /* - * data_size publishes the slot: the reader treats a zero as "the - * producer has claimed this slot but has not filled it yet" and - * waits. Claiming the slot advanced head, so a reader can already - * be looking at it; everything above must be visible first. - */ - smp_store_release(&slot->data_size, data_size); - - mk_arch_send_ipi(target); +int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, + unsigned long type) +{ + struct mk_instance *instance = mk_instance_find(instance_id); + int ret; + if (!instance) + return -EINVAL; + ret = mk_send_ipi_data(instance, data, data_size, type); mk_instance_put(instance); - return 0; + return ret; } -static void mk_ipi_drain_ring(void) +int multikernel_send_ipi_data_to_host(void *data, size_t data_size, + unsigned long type) { - struct mk_ipi_data *slot; - struct mk_ipi_handler *handler; - unsigned int head, tail, next_tail; - size_t data_size; - int messages_processed = 0; - - if (!mk_self || !mk_self->ipi_data) - return; + struct mk_instance *instance = READ_ONCE(host_instance); - while (1) { - tail = mk_ring_idx(atomic_read(&mk_self->ipi_data->ring.tail)); - head = mk_ring_idx(atomic_read(&mk_self->ipi_data->ring.head)); + if (!instance) + return -ENODEV; + return mk_send_ipi_data(instance, data, data_size, type); +} +EXPORT_SYMBOL(multikernel_send_ipi_data_to_host); - if (tail == head) - break; +static void mk_ipi_dispatch(struct mk_ipi_data *slot) +{ + struct mk_ipi_handler *handler; + mk_ipi_callback_t callback = NULL; + void *context = NULL; + unsigned long flags; - slot = &mk_self->ipi_data->ring.entries[tail]; - - /* - * Pairs with the store_release in multikernel_send_ipi_data(). - * Zero means the sender claimed this slot but has not - * finished writing it. Leave it alone: skipping it would - * drop the message it is about to publish. Its own IPI, or - * the next one, brings us back here. - * - * A sender stopped before publishing leaves its slot zero - * forever; mk_ipi_ring_drop_pending() clears those out when - * the instance is re-spawned. - */ - data_size = smp_load_acquire(&slot->data_size); - if (data_size == 0) + if (READ_ONCE(slot->data_size) > MK_MAX_DATA_SIZE) + return; + raw_spin_lock_irqsave(&mk_handlers_lock, flags); + for (handler = mk_handlers; handler; handler = handler->next) { + if (handler->ipi_type == READ_ONCE(slot->type)) { + callback = handler->callback; + context = handler->context; break; - - if (data_size > MK_MAX_DATA_SIZE) { - pr_warn_once("Multikernel IPI slot %u has bad size %zu\n", - tail, data_size); - slot->data_size = 0; - next_tail = mk_ring_idx(tail + 1); - atomic_set(&mk_self->ipi_data->ring.tail, next_tail); - continue; } + } + raw_spin_unlock_irqrestore(&mk_handlers_lock, flags); + if (callback) + callback(slot, context); +} - /* Dispatch to registered handler */ - raw_spin_lock(&mk_handlers_lock); - for (handler = mk_handlers; handler; handler = handler->next) { - if (handler->ipi_type == slot->type && handler->callback) { - mk_ipi_callback_t cb = handler->callback; - void *ctx = handler->context; +static void mk_ipi_drain_endpoint(struct mk_ipi_endpoint *endpoint) +{ + struct mk_ipi_data *slot; + unsigned long flags; + u32 idx; - raw_spin_unlock(&mk_handlers_lock); - cb(slot, ctx); - goto advance_tail; + raw_spin_lock_irqsave(&endpoint->rx_lock, flags); + if (endpoint->rx_dispatching) { + raw_spin_unlock_irqrestore(&endpoint->rx_lock, flags); + return; + } + endpoint->rx_dispatching = true; + raw_spin_unlock_irqrestore(&endpoint->rx_lock, flags); + for (;;) { + idx = endpoint->rx_tail & (MK_IPI_RING_SIZE - 1); + slot = &endpoint->rx->entries[idx]; + /* Pair with the producer's release publication of this slot. */ + if (!smp_load_acquire(&slot->ready)) { + raw_spin_lock_irqsave(&endpoint->rx_lock, flags); + endpoint->rx_dispatching = false; + /* Close the empty-ring handoff race with a new publication. */ + if (smp_load_acquire(&slot->ready)) { + endpoint->rx_dispatching = true; + raw_spin_unlock_irqrestore(&endpoint->rx_lock, flags); + continue; } + raw_spin_unlock_irqrestore(&endpoint->rx_lock, flags); + return; } - raw_spin_unlock(&mk_handlers_lock); - -advance_tail: - /* Mark consumed so the slot reads as unpublished again */ - slot->data_size = 0; - next_tail = mk_ring_idx(tail + 1); - atomic_set(&mk_self->ipi_data->ring.tail, next_tail); - messages_processed++; - - if (messages_processed >= MK_IPI_RING_SIZE) - break; + mk_ipi_dispatch(slot); + /* The callback must finish reading before the slot is reusable. */ + smp_store_release(&slot->ready, 0); + endpoint->rx_tail++; } } -/** - * multikernel_interrupt_handler - Handle the multikernel IPI - * - * This function is called when a multikernel IPI is received. - * Messages are drained here, in interrupt context. - */ -static void multikernel_interrupt_handler(void) +static void mk_ipi_drain_all(void) { - if (!mk_self || !mk_self->ipi_data) + struct mk_ipi_endpoint *endpoint; + + if (!READ_ONCE(mk_handlers_ready)) return; + rcu_read_lock(); + list_for_each_entry_rcu(endpoint, &mk_ipi_endpoints, rx_node) + mk_ipi_drain_endpoint(endpoint); + rcu_read_unlock(); +} - /* - * Drain here rather than from irq_work. We are already in interrupt - * context and every handler is safe to call from it, and irq_work - * brings a failure mode with it: the work is a single static - * instance, so if it is ever left pending - its self-IPI lost while - * the CPU was bringing its APIC up, say - every later queue attempt - * is a no-op and the ring never drains again. - */ - mk_ipi_drain_ring(); +void mk_ipi_handlers_enable(void) +{ + WRITE_ONCE(mk_handlers_ready, true); + mk_ipi_drain_all(); +} + +void mk_poll_ipi_messages(void) +{ + unsigned long flags; + + local_irq_save(flags); + mk_ipi_drain_all(); + local_irq_restore(flags); } -/** - * Generic multikernel interrupt handler - called by the IPI vector - * - * This is the function that gets called by the IPI vector handler. - */ void generic_multikernel_interrupt(void) { - multikernel_interrupt_handler(); + mk_ipi_drain_all(); } /** diff --git a/kernel/multikernel/manifest.c b/kernel/multikernel/manifest.c index e54e9568d0344d..5ee431bf98facb 100644 --- a/kernel/multikernel/manifest.c +++ b/kernel/multikernel/manifest.c @@ -28,12 +28,18 @@ /* Physical address of the manifest this kernel booted with, 0 if none */ static phys_addr_t mk_manifest_fdt_phys; +static bool mk_manifest_fdt_rejected; phys_addr_t mk_manifest_phys(void) { return mk_manifest_fdt_phys; } +bool mk_manifest_rejected(void) +{ + return READ_ONCE(mk_manifest_fdt_rejected); +} + /** * mk_manifest_populate() - Accept the manifest handed over at boot * @fdt_phys: Physical address of the manifest FDT @@ -55,6 +61,7 @@ void __init mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) if (!fdt) { pr_warn("multikernel: failed to memremap manifest (0x%llx)\n", fdt_phys); + err = -ENOMEM; goto out; } @@ -73,14 +80,17 @@ void __init mk_manifest_populate(phys_addr_t fdt_phys, u64 fdt_len) } mk_manifest_fdt_phys = fdt_phys; + mk_manifest_fdt_rejected = false; pr_info("multikernel: manifest accepted\n"); out: if (fdt) early_memunmap(fdt, fdt_len); - if (err) - pr_warn("multikernel: ignoring invalid manifest\n"); + if (err) { + mk_manifest_fdt_rejected = true; + pr_warn("multikernel: supplied manifest rejected: %d\n", err); + } } /* From 2038320dc088bc8ba9ab8d870c58bf4900d70ab8 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Tue, 11 Aug 2026 07:54:21 +0300 Subject: [PATCH 02/24] multikernel: carry assigned PCI resource metadata Describe only the PCI functions assigned to a spawn and retain the BAR shape discovered by the host. Validate exact BDF syntax, reject duplicate functions, and preserve the inventory across the baseline and instance device trees. Keep PCI support behind CONFIG_PCI so the control plane remains buildable without PCI. Signed-off-by: Nikolay Nikolaev --- arch/x86/multikernel/Makefile | 1 + include/linux/multikernel.h | 9 ++ kernel/multikernel/Makefile | 1 + kernel/multikernel/baseline.c | 44 +++++++--- kernel/multikernel/dts.c | 137 ++++++++++++++++++++++++++++--- kernel/multikernel/instance_dt.c | 15 +++- kernel/multikernel/internal.h | 3 +- kernel/multikernel/overlay.c | 14 ++-- kernel/multikernel/pci.c | 77 +++++++++++++++++ 9 files changed, 269 insertions(+), 32 deletions(-) create mode 100644 kernel/multikernel/pci.c diff --git a/arch/x86/multikernel/Makefile b/arch/x86/multikernel/Makefile index 5989cbbb691989..7238cf5d90f337 100644 --- a/arch/x86/multikernel/Makefile +++ b/arch/x86/multikernel/Makefile @@ -4,3 +4,4 @@ # obj-y += spawn.o direct_boot.o head_64.o e820.o +obj-$(CONFIG_PCI) += pci.o diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index 09460a911bb9cc..6beb6330098363 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -585,6 +585,13 @@ struct mk_memory_region { * Format: vendor:device@domain:bus:slot.func */ #define MK_PCI_ALIAS_LEN 32 /* Devicetree spec: alias names are at most 31 chars */ +#define MK_PCI_RESOURCE_COUNT 6 + +struct mk_pci_resource { + u64 start; + u64 end; + u64 flags; +}; /* * A PCI host bridge as the instance device tree describes it: the @@ -602,6 +609,8 @@ struct mk_pci_device { u8 bus; /* PCI bus number */ u8 slot; /* PCI slot number */ u8 func; /* PCI function number */ + struct mk_pci_resource resources[MK_PCI_RESOURCE_COUNT]; + bool resources_valid; struct list_head list; /* Link to device list */ }; diff --git a/kernel/multikernel/Makefile b/kernel/multikernel/Makefile index cf2e1391038ec7..2ff0a209b57c0c 100644 --- a/kernel/multikernel/Makefile +++ b/kernel/multikernel/Makefile @@ -5,6 +5,7 @@ obj-y += core.o cpuset.o mem.o contig.o kernfs.o dts.o instance_dt.o manifest.o ipi.o messaging.o overlay.o hotplug.o baseline.o obj-$(CONFIG_OF) += of_devices.o +obj-$(CONFIG_PCI) += pci.o # DMA-BUF heap for multikernel memory allocation obj-$(CONFIG_DMABUF_HEAPS) += dma_heap.o diff --git a/kernel/multikernel/baseline.c b/kernel/multikernel/baseline.c index 813b26c2677ef2..da561e985cb704 100644 --- a/kernel/multikernel/baseline.c +++ b/kernel/multikernel/baseline.c @@ -238,10 +238,14 @@ static int mk_baseline_parse_devices(const void *fdt, int resources_node, } if (strcmp(device_type, "pci") == 0) { + struct mk_pci_device *existing; struct mk_pci_device *pci_dev; const char *pci_id_str; const fdt32_t *vendor_prop, *device_prop; - unsigned int domain, bus, slot, func; + u32 vendor, device; + u16 domain; + u8 bus, slot, func; + int ret; pci_id_str = fdt_getprop(fdt, dev_node, "pci-id", &len); if (!pci_id_str) { @@ -250,10 +254,12 @@ static int mk_baseline_parse_devices(const void *fdt, int resources_node, return -EINVAL; } - if (sscanf(pci_id_str, "%x:%x:%x.%x", &domain, &bus, &slot, &func) != 4) { - pr_err("Invalid pci-id format '%s' for device '%s'\n", - pci_id_str, dev_name); - return -EINVAL; + ret = mk_pci_parse_bdf(pci_id_str, len, &domain, &bus, + &slot, &func); + if (ret) { + pr_err("Invalid or out-of-range pci-id '%.*s' for device '%s'\n", + len, pci_id_str, dev_name); + return ret; } vendor_prop = fdt_getprop(fdt, dev_node, "vendor-id", &len); @@ -269,6 +275,22 @@ static int mk_baseline_parse_devices(const void *fdt, int resources_node, dev_name); return -EINVAL; } + vendor = fdt32_to_cpu(*vendor_prop); + device = fdt32_to_cpu(*device_prop); + if (vendor > U16_MAX || device > U16_MAX) { + pr_err("Out-of-range vendor-id or device-id for device '%s'\n", + dev_name); + return -ERANGE; + } + list_for_each_entry(existing, pci_list, list) { + if (existing->domain == domain && + existing->bus == bus && existing->slot == slot && + existing->func == func) { + pr_err("Duplicate baseline PCI BDF %04x:%02x:%02x.%x\n", + domain, bus, slot, func); + return -EEXIST; + } + } pci_dev = kzalloc_obj(*pci_dev, GFP_KERNEL); if (!pci_dev) { @@ -279,12 +301,12 @@ static int mk_baseline_parse_devices(const void *fdt, int resources_node, alias = mk_dt_node_alias(fdt, dev_node); if (alias) strscpy(pci_dev->alias, alias, sizeof(pci_dev->alias)); - pci_dev->vendor = (u16)fdt32_to_cpu(*vendor_prop); - pci_dev->device = (u16)fdt32_to_cpu(*device_prop); - pci_dev->domain = (u16)domain; - pci_dev->bus = (u8)bus; - pci_dev->slot = (u8)slot; - pci_dev->func = (u8)func; + pci_dev->vendor = (u16)vendor; + pci_dev->device = (u16)device; + pci_dev->domain = domain; + pci_dev->bus = bus; + pci_dev->slot = slot; + pci_dev->func = func; list_add_tail(&pci_dev->list, pci_list); pci_count++; diff --git a/kernel/multikernel/dts.c b/kernel/multikernel/dts.c index e41a37d6a78254..f537ddd889c006 100644 --- a/kernel/multikernel/dts.c +++ b/kernel/multikernel/dts.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #if defined(CONFIG_X86) && defined(CONFIG_PCI_MMCONFIG) #include @@ -28,6 +29,48 @@ #include "internal.h" +static int mk_pci_parse_hex(const char *str, size_t digits, u32 *value) +{ + size_t i; + u32 parsed = 0; + + for (i = 0; i < digits; i++) { + int digit = hex_to_bin(str[i]); + + if (digit < 0) + return -EINVAL; + parsed = (parsed << 4) | digit; + } + + *value = parsed; + return 0; +} + +int mk_pci_parse_bdf(const char *pci_id, int len, u16 *domain, u8 *bus, + u8 *slot, u8 *func) +{ + u32 parsed_domain, parsed_bus, parsed_slot, parsed_func; + + if (len != (int)sizeof("0000:00:00.0") || pci_id[12] != '\0' || + pci_id[4] != ':' || pci_id[7] != ':' || pci_id[10] != '.') + return -EINVAL; + + if (mk_pci_parse_hex(pci_id, 4, &parsed_domain) || + mk_pci_parse_hex(pci_id + 5, 2, &parsed_bus) || + mk_pci_parse_hex(pci_id + 8, 2, &parsed_slot) || + mk_pci_parse_hex(pci_id + 11, 1, &parsed_func)) + return -EINVAL; + if (parsed_domain > U16_MAX || parsed_bus > U8_MAX || + parsed_slot > 31 || parsed_func > 7) + return -ERANGE; + + *domain = (u16)parsed_domain; + *bus = (u8)parsed_bus; + *slot = (u8)parsed_slot; + *func = (u8)parsed_func; + return 0; +} + /** * mk_dt_node_alias() - Find the /aliases entry naming a node * @fdt: Device tree blob @@ -263,9 +306,12 @@ static int mk_dt_add_pci_device(const void *source_fdt, int dev_node, unsigned int func) { const fdt32_t *vendor_prop, *device_prop; + const fdt64_t *resources_prop; + struct mk_pci_device *existing; struct mk_pci_device *pci_dev; const char *node_name, *alias; - int len; + u32 vendor, device; + int len, i; node_name = fdt_get_name(source_fdt, dev_node, NULL); @@ -282,6 +328,16 @@ static int mk_dt_add_pci_device(const void *source_fdt, int dev_node, device_name, node_name ? node_name : ""); return -EINVAL; } + vendor = fdt32_to_cpu(*vendor_prop); + device = fdt32_to_cpu(*device_prop); + if (vendor > U16_MAX || device > U16_MAX || domain > U16_MAX || + bus > U8_MAX || slot > 31 || func > 7) + return -ERANGE; + list_for_each_entry(existing, &config->pci_devices, list) { + if (existing->domain == domain && existing->bus == bus && + existing->slot == slot && existing->func == func) + return -EEXIST; + } pci_dev = kzalloc(sizeof(*pci_dev), GFP_KERNEL); if (!pci_dev) { @@ -289,8 +345,8 @@ static int mk_dt_add_pci_device(const void *source_fdt, int dev_node, return -ENOMEM; } - pci_dev->vendor = (u16)fdt32_to_cpu(*vendor_prop); - pci_dev->device = (u16)fdt32_to_cpu(*device_prop); + pci_dev->vendor = (u16)vendor; + pci_dev->device = (u16)device; pci_dev->domain = (u16)domain; pci_dev->bus = (u8)bus; pci_dev->slot = (u8)slot; @@ -298,6 +354,28 @@ static int mk_dt_add_pci_device(const void *source_fdt, int dev_node, alias = mk_dt_node_alias(source_fdt, dev_node); if (alias) strscpy(pci_dev->alias, alias, sizeof(pci_dev->alias)); + resources_prop = fdt_getprop(source_fdt, dev_node, "bar-resources", &len); + if (resources_prop) { + if (len != MK_PCI_RESOURCE_COUNT * 3 * sizeof(*resources_prop)) { + kfree(pci_dev); + return -EINVAL; + } + for (i = 0; i < MK_PCI_RESOURCE_COUNT; i++) { + u64 start = fdt64_to_cpu(resources_prop[i * 3]); + u64 end = fdt64_to_cpu(resources_prop[i * 3 + 1]); + u64 flags = fdt64_to_cpu(resources_prop[i * 3 + 2]); + + if ((start || end) && + (end < start || !(flags & (IORESOURCE_IO | IORESOURCE_MEM)))) { + kfree(pci_dev); + return -EINVAL; + } + pci_dev->resources[i].start = start; + pci_dev->resources[i].end = end; + pci_dev->resources[i].flags = flags; + } + pci_dev->resources_valid = true; + } list_add_tail(&pci_dev->list, &config->pci_devices); config->pci_device_count++; @@ -313,9 +391,10 @@ static int mk_dt_parse_single_pci_device(const void *source_fdt, int dev_node, struct mk_dt_config *config, const char *device_name) { - unsigned int domain, bus, slot, func; + u16 domain; + u8 bus, slot, func; const char *pci_id_str; - int len; + int len, ret; pci_id_str = fdt_getprop(source_fdt, dev_node, "pci-id", &len); if (!pci_id_str) { @@ -323,11 +402,9 @@ static int mk_dt_parse_single_pci_device(const void *source_fdt, int dev_node, return -EINVAL; } - if (sscanf(pci_id_str, "%x:%x:%x.%x", &domain, &bus, &slot, &func) != 4) { - pr_err("Invalid pci-id format: '%s' (expected domain:bus:slot.func)\n", - pci_id_str); - return -EINVAL; - } + ret = mk_pci_parse_bdf(pci_id_str, len, &domain, &bus, &slot, &func); + if (ret) + return ret; return mk_dt_add_pci_device(source_fdt, dev_node, config, device_name, domain, bus, slot, func); @@ -1096,6 +1173,44 @@ static struct pci_bus *mk_dt_bus_toward(struct pci_bus *parent, u16 domain, return bus; } +static int mk_dt_emit_pci_resources(void *fdt, + const struct mk_pci_device *device) +{ + fdt64_t resources[MK_PCI_RESOURCE_COUNT * 3]; + struct pci_dev *live_dev = NULL; + int i, ret; + + if (!device->resources_valid) { + live_dev = pci_get_domain_bus_and_slot(device->domain, device->bus, + PCI_DEVFN(device->slot, + device->func)); + if (!live_dev) + return -ENODEV; + } + + for (i = 0; i < MK_PCI_RESOURCE_COUNT; i++) { + u64 start, end, flags; + + if (live_dev) { + start = pci_resource_start(live_dev, i); + end = pci_resource_end(live_dev, i); + flags = pci_resource_flags(live_dev, i); + } else { + start = device->resources[i].start; + end = device->resources[i].end; + flags = device->resources[i].flags; + } + resources[i * 3] = cpu_to_fdt64(start); + resources[i * 3 + 1] = cpu_to_fdt64(end); + resources[i * 3 + 2] = cpu_to_fdt64(flags); + } + + if (live_dev) + pci_dev_put(live_dev); + ret = fdt_property(fdt, "bar-resources", resources, sizeof(resources)); + return ret; +} + /* * Describe what @instance owns below @bus: its devices on this bus as * leaves, and one bridge node, recursed into, per child bus that leads @@ -1124,6 +1239,8 @@ static int mk_dt_emit_pci_bus(struct mk_instance *instance, ret = fdt_property_u32(fdt, "vendor-id", dev->vendor); if (!ret) ret = fdt_property_u32(fdt, "device-id", dev->device); + if (!ret) + ret = mk_dt_emit_pci_resources(fdt, dev); if (!ret) ret = fdt_end_node(fdt); if (ret) diff --git a/kernel/multikernel/instance_dt.c b/kernel/multikernel/instance_dt.c index cd4d437ed84137..835922bc491075 100644 --- a/kernel/multikernel/instance_dt.c +++ b/kernel/multikernel/instance_dt.c @@ -279,6 +279,17 @@ static int __init mk_copy_platform_devices(const struct mk_dt_config *config, return 0; } +static void __init mk_take_pci_devices(struct mk_dt_config *config, + struct mk_instance *instance) +{ + list_splice_tail_init(&config->pci_devices, &instance->pci_devices); + instance->pci_device_count = config->pci_device_count; + instance->pci_devices_valid = config->pci_devices_valid; + + config->pci_device_count = 0; + config->pci_devices_valid = false; +} + /* A message ring the host describes in /chosen: its address and size */ static bool __init mk_chosen_ring(const char *what, phys_addr_t *phys, u32 *pages) @@ -528,8 +539,8 @@ int __init mk_instance_restore_from_manifest(void) goto cleanup_instance; } - /* The tree is the record of this kernel's PCI devices; the list stays empty */ - instance->pci_devices_valid = true; + /* Config entries become this kernel's assigned-device allowlist. */ + mk_take_pci_devices(&config, instance); ret = mk_copy_platform_devices(&config, instance); if (ret) { diff --git a/kernel/multikernel/internal.h b/kernel/multikernel/internal.h index fa1c015b936e3b..ba086d95ca2942 100644 --- a/kernel/multikernel/internal.h +++ b/kernel/multikernel/internal.h @@ -43,7 +43,8 @@ int mk_dt_emit_boot_tree(struct mk_instance *instance, void *fdt, size_t size, int (*chosen)(void *fdt, void *data), void *data); int mk_dt_generate_instance_dtb(struct mk_instance *instance, void **out_dtb, size_t *out_size); - +int mk_pci_parse_bdf(const char *pci_id, int len, u16 *domain, u8 *bus, + u8 *slot, u8 *func); /* overlay.c */ extern struct kernfs_node *mk_overlay_root_kn; extern struct mutex mk_overlay_mutex; diff --git a/kernel/multikernel/overlay.c b/kernel/multikernel/overlay.c index 3ee26597620c57..cf8d7ad82951e8 100644 --- a/kernel/multikernel/overlay.c +++ b/kernel/multikernel/overlay.c @@ -565,9 +565,9 @@ static int mk_overlay_parse_pci_id(const void *fdt, int item_node, const char *name, u16 *domain, u8 *bus, u8 *devfn) { - unsigned int d, b, slot, func; + u8 slot, func; const char *pci_id_str; - int len; + int len, ret; pci_id_str = fdt_getprop(fdt, item_node, "pci-id", &len); if (!pci_id_str || len <= 0 || pci_id_str[len - 1] != '\0') { @@ -575,13 +575,11 @@ static int mk_overlay_parse_pci_id(const void *fdt, int item_node, return -EINVAL; } - if (sscanf(pci_id_str, "%x:%x:%x.%x", &d, &b, &slot, &func) != 4) { - pr_err("Invalid pci-id format '%s'\n", pci_id_str); - return -EINVAL; + ret = mk_pci_parse_bdf(pci_id_str, len, domain, bus, &slot, &func); + if (ret) { + pr_err("Invalid or out-of-range pci-id '%s'\n", pci_id_str); + return ret; } - - *domain = (u16)d; - *bus = (u8)b; *devfn = PCI_DEVFN(slot, func); return 0; } diff --git a/kernel/multikernel/pci.c b/kernel/multikernel/pci.c new file mode 100644 index 00000000000000..61c5f9cbe15e10 --- /dev/null +++ b/kernel/multikernel/pci.c @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Multikernel PCI assignment policy + * + * Keeps assigned-device discovery, identity presentation, bridge traversal, + * and resource restoration independent from the manifest that populated + * the current instance. + */ + +#include +#include + +#include "internal.h" + +static struct mk_pci_device *mk_pci_find_assigned(struct pci_bus *bus, int devfn) +{ + struct mk_pci_device *device; + u16 domain = pci_domain_nr(bus); + u8 slot = PCI_SLOT(devfn); + u8 func = PCI_FUNC(devfn); + + if (!root_instance || !root_instance->dtb_data || + !root_instance->pci_devices_valid) + return NULL; + + list_for_each_entry(device, &root_instance->pci_devices, list) { + if (device->domain == domain && device->bus == bus->number && + device->slot == slot && device->func == func) + return device; + } + + return NULL; +} + +/** + * mk_pci_get_assigned_identity - Get the identity presented to an instance + * @bus: PCI bus + * @devfn: device/function number + * @vendor: assigned Vendor ID + * @device_id: assigned Device ID + * + * Returns: true when assignment metadata contains an exact location match. + */ +bool mk_pci_get_assigned_identity(struct pci_bus *bus, int devfn, + u16 *vendor, u16 *device_id) +{ + struct mk_pci_device *device = mk_pci_find_assigned(bus, devfn); + + if (!device) + return false; + + if (vendor) + *vendor = device->vendor; + if (device_id) + *device_id = device->device; + return true; +} + +static void mk_pci_restore_resources(struct pci_dev *dev) +{ + struct mk_pci_device *device; + int i; + + device = mk_pci_find_assigned(dev->bus, dev->devfn); + if (!device || !device->resources_valid) + return; + + dev->non_compliant_bars = true; + for (i = 0; i < MK_PCI_RESOURCE_COUNT; i++) { + dev->resource[i].start = device->resources[i].start; + dev->resource[i].end = device->resources[i].end; + dev->resource[i].flags = device->resources[i].flags; + } + pr_info("Restored PCI BAR resources for %s\n", pci_name(dev)); +} + +DECLARE_PCI_FIXUP_EARLY(PCI_ANY_ID, PCI_ANY_ID, mk_pci_restore_resources); From e355d712a2b0c31421ea4fb8331a4340ac0949fc Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Tue, 11 Aug 2026 07:55:41 +0300 Subject: [PATCH 03/24] x86/multikernel: build PCI roots for assigned functions Create one synthetic root bus for each assigned domain and bus, then scan only the explicitly assigned devfns. Supply an exact bus-number resource when x86 root resources do not provide one. Spawn kernels do not inherit or map host ECAM windows. Configuration is mediated by the filtered backend introduced with the assigned roots. Signed-off-by: Nikolay Nikolaev --- arch/x86/include/asm/multikernel.h | 6 ++ arch/x86/kernel/platform-quirks.c | 5 +- arch/x86/multikernel/pci.c | 152 +++++++++++++++++++++++++++++ drivers/pci/probe.c | 9 -- 4 files changed, 159 insertions(+), 13 deletions(-) create mode 100644 arch/x86/multikernel/pci.c diff --git a/arch/x86/include/asm/multikernel.h b/arch/x86/include/asm/multikernel.h index 953fe71fc044a7..2911e803d9c25c 100644 --- a/arch/x86/include/asm/multikernel.h +++ b/arch/x86/include/asm/multikernel.h @@ -11,6 +11,7 @@ #ifndef __ASSEMBLY__ #include +#include #include #include #include @@ -222,6 +223,11 @@ int multikernel_wakeup_secondary_cpu_64(u32 apicid, unsigned long start_eip, int multikernel_restore_ap(unsigned int cpu, unsigned long cr3, unsigned long gs_base, unsigned long stack, unsigned long entry); +#if defined(CONFIG_MULTIKERNEL) && defined(CONFIG_PCI) +void __init x86_multikernel_pci_platform_init(void); +#else +static inline void x86_multikernel_pci_platform_init(void) { } +#endif /* NMI on an offline pool CPU: honor a pending force halt */ #ifdef CONFIG_MULTIKERNEL diff --git a/arch/x86/kernel/platform-quirks.c b/arch/x86/kernel/platform-quirks.c index fcd674b933102c..ca677416a1d805 100644 --- a/arch/x86/kernel/platform-quirks.c +++ b/arch/x86/kernel/platform-quirks.c @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include @@ -171,6 +170,7 @@ void __init x86_early_init_platform_quirks(void) break; case X86_SUBARCH_MULTIKERNEL: multikernel_setup_calibration(); + x86_multikernel_pci_platform_init(); x86_platform.legacy.devices.pnpbios = 0; x86_platform.legacy.i8042 = X86_LEGACY_I8042_PLATFORM_ABSENT; x86_platform.legacy.rtc = 0; @@ -194,9 +194,6 @@ void __init x86_early_init_platform_quirks(void) x86_init.paging.pagetable_init = multikernel_pagetable_init; #endif x86_init.mpparse.parse_smp_cfg = multikernel_parse_smp_config; -#ifdef CONFIG_PCI - x86_init.pci.init = pci_multikernel_init; -#endif #endif x86_init.mpparse.early_parse_smp_cfg = x86_init_noop; /* diff --git a/arch/x86/multikernel/pci.c b/arch/x86/multikernel/pci.c new file mode 100644 index 00000000000000..85d2dba737acd7 --- /dev/null +++ b/arch/x86/multikernel/pci.c @@ -0,0 +1,152 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * x86 PCI support for multikernel spawn kernels. + * + * Spawn kernels discover only assigned BDFs. Configuration accesses are + * filtered here and become host-mediated once the RPC transport is installed. + */ +#include +#include +#include +#include +#include + +#include +#include + +static struct pci_ops mk_pci_native_ops; +static bool mk_pci_roots_ready; + +static bool mk_pci_identity_read(u16 vendor, u16 device, int where, int size, + u32 *value) +{ + u32 identity; + u32 mask; + + if (where < PCI_VENDOR_ID || where + size > PCI_COMMAND) + return false; + identity = vendor | (u32)device << 16; + mask = size == sizeof(identity) ? ~0U : (1U << (size * 8)) - 1; + *value = (identity >> (where * 8)) & mask; + return true; +} + +static int mk_pci_read(struct pci_bus *bus, unsigned int devfn, int where, + int size, u32 *value) +{ + u16 vendor, device; + + if (!mk_pci_get_assigned_identity(bus, devfn, &vendor, &device)) { + *value = ~0U; + return PCIBIOS_DEVICE_NOT_FOUND; + } + if (mk_pci_identity_read(vendor, device, where, size, value)) + return PCIBIOS_SUCCESSFUL; + return mk_pci_native_ops.read(bus, devfn, where, size, value); +} + +static int mk_pci_write(struct pci_bus *bus, unsigned int devfn, int where, + int size, u32 value) +{ + if (!mk_pci_get_assigned_identity(bus, devfn, NULL, NULL)) + return PCIBIOS_DEVICE_NOT_FOUND; + return mk_pci_native_ops.write(bus, devfn, where, size, value); +} + +static int __init x86_multikernel_pci_arch_init(void) +{ + if (!root_instance || !root_instance->pci_devices_valid) + return 0; + + mk_pci_native_ops = pci_root_ops; + pci_root_ops.read = mk_pci_read; + pci_root_ops.write = mk_pci_write; + mk_pci_roots_ready = true; + pr_notice("Multikernel selected filtered PCI config access\n"); + return 0; +} + +static struct pci_bus * __init mk_pci_get_root(u16 domain, u8 bus_number) +{ + struct resource_entry *window; + struct pci_sysdata *sd; + struct pci_bus *bus; + bool has_busn_res = false; + LIST_HEAD(resources); + + if (domain && !pci_domains_supported) { + pr_err("Multikernel cannot scan PCI root %04x:%02x without domain support\n", + domain, bus_number); + return ERR_PTR(-EOPNOTSUPP); + } + bus = pci_find_bus(domain, bus_number); + if (bus) + return bus; + + sd = kzalloc_obj(*sd, GFP_KERNEL); + if (!sd) + return ERR_PTR(-ENOMEM); + sd->domain = domain; + sd->node = x86_pci_root_bus_node(bus_number); + x86_pci_root_bus_resources(bus_number, &resources); + resource_list_for_each_entry(window, &resources) { + if (window->res->flags & IORESOURCE_BUS) { + has_busn_res = true; + break; + } + } + bus = pci_create_root_bus(NULL, bus_number, &pci_root_ops, sd, + &resources); + if (!bus) { + pci_free_resource_list(&resources); + kfree(sd); + return ERR_PTR(-ENOMEM); + } + if (!has_busn_res && + !pci_bus_insert_busn_res(bus, bus_number, bus_number)) { + pci_remove_root_bus(bus); + kfree(sd); + return ERR_PTR(-EBUSY); + } + pr_notice("Multikernel created synthetic PCI root %04x:%02x\n", + domain, bus_number); + return bus; +} + +static int __init x86_multikernel_pci_init(void) +{ + const struct mk_pci_device *device; + struct pci_bus *bus; + struct pci_dev *pdev; + + if (!root_instance) + panic("Multikernel lost restored instance metadata"); + if (!root_instance->pci_device_count) + return 0; + if (!root_instance->pci_devices_valid || !mk_pci_roots_ready) + panic("Multikernel assigned PCI inventory is unavailable"); + + list_for_each_entry(device, &root_instance->pci_devices, list) { + bus = mk_pci_get_root(device->domain, device->bus); + if (IS_ERR(bus)) + panic("Multikernel failed to create synthetic PCI root %04x:%02x: %ld", + device->domain, device->bus, PTR_ERR(bus)); + pdev = pci_scan_single_device(bus, + PCI_DEVFN(device->slot, device->func)); + if (!pdev) + panic("Multikernel failed to enumerate assigned PCI device %04x:%02x:%02x.%x", + device->domain, device->bus, device->slot, + device->func); + pci_bus_add_devices(bus); + } + + /* Suppress legacy bus 0 probing after every assigned function is present. */ + return 0; +} + +void __init x86_multikernel_pci_platform_init(void) +{ + pci_probe = PCI_PROBE_NOEARLY; + x86_init.pci.arch_init = x86_multikernel_pci_arch_init; + x86_init.pci.init = x86_multikernel_pci_init; +} diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index 7661517ebdf0df..bccc7a4bdd7943 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -24,7 +24,6 @@ #include #include #include -#include #include "pci.h" static struct resource busn_resource = { @@ -2601,14 +2600,6 @@ static struct pci_dev *pci_scan_device(struct pci_bus *bus, int devfn) struct pci_dev *dev; u32 l; - /* - * For multikernel spawns, check if we should even probe this location - * BEFORE any config space access. This prevents hardware conflicts - * when the host kernel is also using PCI devices. - */ - if (IS_ENABLED(CONFIG_MULTIKERNEL) && !mk_pci_should_probe(bus, devfn)) - return NULL; - if (!pci_bus_read_dev_vendor_id(bus, devfn, &l, 60*1000)) return NULL; From 6c6d1d2fa83935dbf074a37b839fb201f1d66783 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Tue, 4 Aug 2026 08:26:40 +0300 Subject: [PATCH 04/24] x86/multikernel: filter raw PCI config operations Synthetic root filtering alone leaves raw x86 configuration backends able to reach functions outside the spawn inventory. Apply the assigned-BDF filter to both raw configuration entry points. Identity reads come from validated metadata while other accesses use the selected backend only for an assigned function. Signed-off-by: Nikolay Nikolaev --- arch/x86/multikernel/pci.c | 70 ++++++++++++++++++++++++++++++------- include/linux/multikernel.h | 36 +++++++++++++++---- kernel/multikernel/pci.c | 27 +++++++++----- 3 files changed, 105 insertions(+), 28 deletions(-) diff --git a/arch/x86/multikernel/pci.c b/arch/x86/multikernel/pci.c index 85d2dba737acd7..2a3d9a84375386 100644 --- a/arch/x86/multikernel/pci.c +++ b/arch/x86/multikernel/pci.c @@ -14,7 +14,8 @@ #include #include -static struct pci_ops mk_pci_native_ops; +static const struct pci_raw_ops *mk_pci_native_raw_ops; +static const struct pci_raw_ops *mk_pci_native_raw_ext_ops; static bool mk_pci_roots_ready; static bool mk_pci_identity_read(u16 vendor, u16 device, int where, int size, @@ -31,38 +32,81 @@ static bool mk_pci_identity_read(u16 vendor, u16 device, int where, int size, return true; } -static int mk_pci_read(struct pci_bus *bus, unsigned int devfn, int where, - int size, u32 *value) +static int mk_pci_raw_read(const struct pci_raw_ops *native, + unsigned int domain, unsigned int bus, + unsigned int devfn, int where, int size, + u32 *value) { u16 vendor, device; - if (!mk_pci_get_assigned_identity(bus, devfn, &vendor, &device)) { + if (!mk_pci_get_assigned_identity_bdf(domain, bus, devfn, &vendor, + &device)) { *value = ~0U; return PCIBIOS_DEVICE_NOT_FOUND; } if (mk_pci_identity_read(vendor, device, where, size, value)) return PCIBIOS_SUCCESSFUL; - return mk_pci_native_ops.read(bus, devfn, where, size, value); + return native->read(domain, bus, devfn, where, size, value); } -static int mk_pci_write(struct pci_bus *bus, unsigned int devfn, int where, - int size, u32 value) +static int mk_pci_raw_write(const struct pci_raw_ops *native, + unsigned int domain, unsigned int bus, + unsigned int devfn, int where, int size, + u32 value) { - if (!mk_pci_get_assigned_identity(bus, devfn, NULL, NULL)) + if (!mk_pci_get_assigned_identity_bdf(domain, bus, devfn, NULL, NULL)) return PCIBIOS_DEVICE_NOT_FOUND; - return mk_pci_native_ops.write(bus, devfn, where, size, value); + return native->write(domain, bus, devfn, where, size, value); } +static int mk_pci_read(unsigned int domain, unsigned int bus, + unsigned int devfn, int where, int size, u32 *value) +{ + return mk_pci_raw_read(mk_pci_native_raw_ops, domain, bus, devfn, + where, size, value); +} + +static int mk_pci_write(unsigned int domain, unsigned int bus, + unsigned int devfn, int where, int size, u32 value) +{ + return mk_pci_raw_write(mk_pci_native_raw_ops, domain, bus, devfn, + where, size, value); +} + +static int mk_pci_ext_read(unsigned int domain, unsigned int bus, + unsigned int devfn, int where, int size, u32 *value) +{ + return mk_pci_raw_read(mk_pci_native_raw_ext_ops, domain, bus, devfn, + where, size, value); +} + +static int mk_pci_ext_write(unsigned int domain, unsigned int bus, + unsigned int devfn, int where, int size, u32 value) +{ + return mk_pci_raw_write(mk_pci_native_raw_ext_ops, domain, bus, devfn, + where, size, value); +} + +static const struct pci_raw_ops mk_pci_filtered_raw_ops = { + .read = mk_pci_read, + .write = mk_pci_write, +}; + +static const struct pci_raw_ops mk_pci_filtered_raw_ext_ops = { + .read = mk_pci_ext_read, + .write = mk_pci_ext_write, +}; static int __init x86_multikernel_pci_arch_init(void) { if (!root_instance || !root_instance->pci_devices_valid) return 0; - mk_pci_native_ops = pci_root_ops; - pci_root_ops.read = mk_pci_read; - pci_root_ops.write = mk_pci_write; + mk_pci_native_raw_ops = raw_pci_ops; + mk_pci_native_raw_ext_ops = raw_pci_ext_ops; + raw_pci_ops = &mk_pci_filtered_raw_ops; + raw_pci_ext_ops = &mk_pci_filtered_raw_ext_ops; mk_pci_roots_ready = true; - pr_notice("Multikernel selected filtered PCI config access\n"); + pr_notice("Multikernel selected filtered raw PCI config access\n"); return 0; } diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index 6beb6330098363..a6d7a2a0d1c11d 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -980,8 +980,11 @@ int mk_instance_abort_spawn(struct mk_instance *instance); void *mk_kimage_alloc(struct kimage *image, size_t size, size_t align); void mk_kimage_free(struct kimage *image, void *virt_addr, size_t size); -/* Device probe filtering against the instance's allowlist */ +/* Device filtering against the current kernel's boot tree */ bool mk_pci_should_probe(struct pci_bus *bus, int devfn); +bool mk_pci_get_assigned_identity_bdf(unsigned int domain, unsigned int bus, + unsigned int devfn, u16 *vendor, + u16 *device); bool mk_platform_device_allowed(const char *name, const char *hid); /* Early CPU registration from the manifest (spawn kernels) */ @@ -1035,6 +1038,14 @@ static inline bool mk_pci_should_probe(struct pci_bus *bus, int devfn) { return true; } + +static inline bool +mk_pci_get_assigned_identity_bdf(unsigned int domain, unsigned int bus, + unsigned int devfn, u16 *vendor, u16 *device) +{ + return false; +} + static inline bool mk_platform_device_allowed(const char *name, const char *hid) { return true; @@ -1092,7 +1103,7 @@ int __init mk_instance_restore_from_manifest(void); */ /** - * PCI Device Enforcement Functions + * PCI Device Filtering Functions */ /** @@ -1100,13 +1111,24 @@ int __init mk_instance_restore_from_manifest(void); * @bus: PCI bus * @devfn: PCI device/function number * - * Called BEFORE any PCI config space reads to determine if probing - * should proceed. This prevents config space accesses to devices - * that are not in the whitelist, avoiding hardware conflicts on bare metal. + * Returns: true if probing should proceed, false to skip entirely. + */ + +/** + * mk_pci_get_assigned_identity_bdf() - Identify an assigned PCI function + * @domain: PCI domain number + * @bus: PCI bus number + * @devfn: PCI device/function number + * @vendor: optional assigned vendor ID output + * @device: optional assigned device ID output * - * Returns: true if probing should proceed, false to skip entirely + * The raw x86 PCI configuration wrappers use this BDF-only lookup before + * reaching their hardware backend. Spawn kernels accept exact functions + * recorded in their boot tree; a host kernel without a parent keeps normal + * PCI discovery. A privileged spawn kernel can bypass these wrappers, so the + * check prevents accidental access rather than isolating a hostile kernel. * - * Declared above with the CONFIG_MULTIKERNEL stubs. + * Returns: true for an exact assignment match, false otherwise. */ /** diff --git a/kernel/multikernel/pci.c b/kernel/multikernel/pci.c index 61c5f9cbe15e10..23b713d9aeedb9 100644 --- a/kernel/multikernel/pci.c +++ b/kernel/multikernel/pci.c @@ -12,10 +12,10 @@ #include "internal.h" -static struct mk_pci_device *mk_pci_find_assigned(struct pci_bus *bus, int devfn) +static struct mk_pci_device * +mk_pci_find_assigned_bdf(u16 domain, u8 bus, u8 devfn) { struct mk_pci_device *device; - u16 domain = pci_domain_nr(bus); u8 slot = PCI_SLOT(devfn); u8 func = PCI_FUNC(devfn); @@ -24,7 +24,7 @@ static struct mk_pci_device *mk_pci_find_assigned(struct pci_bus *bus, int devfn return NULL; list_for_each_entry(device, &root_instance->pci_devices, list) { - if (device->domain == domain && device->bus == bus->number && + if (device->domain == domain && device->bus == bus && device->slot == slot && device->func == func) return device; } @@ -32,19 +32,30 @@ static struct mk_pci_device *mk_pci_find_assigned(struct pci_bus *bus, int devfn return NULL; } +static struct mk_pci_device *mk_pci_find_assigned(struct pci_bus *bus, int devfn) +{ + return mk_pci_find_assigned_bdf(pci_domain_nr(bus), bus->number, devfn); +} + /** - * mk_pci_get_assigned_identity - Get the identity presented to an instance - * @bus: PCI bus + * mk_pci_get_assigned_identity_bdf - Get an assigned function's identity + * @domain: PCI domain number + * @bus: PCI bus number * @devfn: device/function number * @vendor: assigned Vendor ID * @device_id: assigned Device ID * * Returns: true when assignment metadata contains an exact location match. */ -bool mk_pci_get_assigned_identity(struct pci_bus *bus, int devfn, - u16 *vendor, u16 *device_id) +bool mk_pci_get_assigned_identity_bdf(unsigned int domain, unsigned int bus, + unsigned int devfn, u16 *vendor, + u16 *device_id) { - struct mk_pci_device *device = mk_pci_find_assigned(bus, devfn); + struct mk_pci_device *device; + + if (domain != (u16)domain || bus != (u8)bus || devfn != (u8)devfn) + return false; + device = mk_pci_find_assigned_bdf(domain, bus, devfn); if (!device) return false; From 4c197fedb7a0bef75fb99ec797181d21f27aa22f Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Thu, 30 Jul 2026 08:17:44 +0300 Subject: [PATCH 05/24] pci/multikernel: add exclusive SR-IOV VF leases Reserve assigned VFs transactionally under a host-wide lease lock. Detach the host driver, mark the device assigned, and publish ownership only when the complete request succeeds. Propagate host-driver restoration failures and retain failed instance state when cleanup cannot be completed safely. Signed-off-by: Nikolay Nikolaev --- Documentation/multikernel/usage.rst | 14 + include/linux/multikernel.h | 2 + kernel/multikernel/baseline.c | 17 +- kernel/multikernel/core.c | 251 +++---------- kernel/multikernel/internal.h | 14 + kernel/multikernel/kernfs.c | 9 + kernel/multikernel/pci.c | 564 +++++++++++++++++++++++++++- 7 files changed, 657 insertions(+), 214 deletions(-) diff --git a/Documentation/multikernel/usage.rst b/Documentation/multikernel/usage.rst index 8ed85d8ead7e18..ea153740658173 100644 --- a/Documentation/multikernel/usage.rst +++ b/Documentation/multikernel/usage.rst @@ -66,6 +66,20 @@ Workflow ``device-remove``, and destroy the instance with ``instance-remove`` once it is stopped. +SR-IOV Assignment Boundary +=========================== + +SR-IOV assignment is intended for cooperative spawned kernels. Filtering +configuration-space access and enumerating only assigned BDFs prevents +accidental access; it is not a security boundary against a privileged kernel +that deliberately issues configuration cycles or maps host physical windows. + +The host keeps exclusive leases for assigned VFs. Assignment fails unless the +device is an SR-IOV VF whose identity and ownership can be validated. Host +drivers remain bound while a VF is merely in the pool and are detached only +while committing a lease. Nested kernels cannot establish this host-owned +lifecycle and therefore cannot lease PCI devices to their children. + Instance States =============== diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index a6d7a2a0d1c11d..685c6220c5187a 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -691,6 +691,8 @@ struct mk_instance { struct list_head pci_devices; /* List of struct mk_pci_device */ int pci_device_count; /* Number of PCI devices */ bool pci_devices_valid; /* Whether PCI device list is valid */ + /* Host-only live PCI assignment leases (private elements). */ + struct list_head pci_assignments; /* Platform device resources */ struct list_head platform_devices; /* List of struct mk_platform_device */ diff --git a/kernel/multikernel/baseline.c b/kernel/multikernel/baseline.c index da561e985cb704..5921f3d4f7670b 100644 --- a/kernel/multikernel/baseline.c +++ b/kernel/multikernel/baseline.c @@ -406,7 +406,7 @@ static int mk_baseline_initialize_cpus(const struct mk_cpu_set *requested) static int mk_baseline_initialize_devices(struct list_head *pci_list) { struct mk_pci_device *pci_dev; - int ret, failed = 0, moved = 0; + int ret, failed = 0, available = 0; if (list_empty(pci_list)) { pr_debug("No PCI devices in baseline to move into the pool\n"); @@ -414,24 +414,25 @@ static int mk_baseline_initialize_devices(struct list_head *pci_list) } list_for_each_entry(pci_dev, pci_list, list) { - ret = mk_pool_device_add(pci_dev->domain, pci_dev->bus, - PCI_DEVFN(pci_dev->slot, pci_dev->func), - pci_dev->alias); + ret = mk_root_add_pci_device(pci_dev->domain, pci_dev->bus, + PCI_DEVFN(pci_dev->slot, pci_dev->func), + pci_dev->alias); if (ret) { - pr_warn("PCI device %04x:%04x@%04x:%02x:%02x.%x not moved into the pool: %d\n", + pr_warn("PCI device %04x:%04x@%04x:%02x:%02x.%x unavailable for the pool: %d\n", pci_dev->vendor, pci_dev->device, pci_dev->domain, pci_dev->bus, pci_dev->slot, pci_dev->func, ret); failed++; continue; } - moved++; + available++; } if (failed > 0) - pr_warn("Failed to move %d PCI devices into the pool\n", failed); + pr_warn("Failed to inventory %d PCI devices for the pool\n", failed); - pr_info("Moved %d PCI devices into the multikernel pool\n", moved); + pr_info("Inventoried %d PCI devices; host drivers remain bound until assignment\n", + available); return 0; } diff --git a/kernel/multikernel/core.c b/kernel/multikernel/core.c index cecdebdcc36d6c..27f554a5827379 100644 --- a/kernel/multikernel/core.c +++ b/kernel/multikernel/core.c @@ -21,22 +21,26 @@ static void mk_instance_return_all_cpus(struct mk_instance *instance) if (!instance || mk_cpu_set_empty(instance->cpus)) return; - if (instance == mk_self || instance->id == 0) + if (!instance || instance == mk_self || instance->id == 0) return; mk_instance_return_cpus(instance, instance->cpus); } -static void mk_instance_return_pci_devices(struct mk_instance *instance) +static int mk_instance_return_pci_devices(struct mk_instance *instance) { struct mk_pci_device *pci_dev, *pci_tmp; int returned_count = 0; + int ret; - if (!instance || !instance->pci_devices_valid) - return; + if (!instance || instance == mk_self || instance->id == 0) + return 0; - if (instance == mk_self || instance->id == 0) - return; + ret = mk_pci_release_assignments(instance); + if (ret) + return ret; + if (!instance || !instance->pci_devices_valid) + return 0; if (!mk_self) { pr_warn("Cannot return PCI devices from instance %d (%s): no self instance\n", @@ -88,6 +92,7 @@ static void mk_instance_return_pci_devices(struct mk_instance *instance) } instance->pci_device_count = 0; instance->pci_devices_valid = false; + return 0; } static void mk_instance_return_platform_devices(struct mk_instance *instance) @@ -165,19 +170,36 @@ static void mk_instance_track_dump(struct mk_instance *instance, atomic_dec(&mk_dump_backups); } +int mk_instance_release_resources(struct mk_instance *instance) +{ + int ret; + + if (!instance || instance == mk_self || instance->id == 0) + return 0; + ret = mk_instance_return_pci_devices(instance); + if (ret) + return ret; + mk_instance_return_platform_devices(instance); + mk_instance_return_all_cpus(instance); + mk_instance_free_memory(instance); + return 0; +} + static void mk_instance_release(struct kref *kref) { struct mk_instance *instance = container_of(kref, struct mk_instance, refcount); + int ret; pr_info("Releasing multikernel instance %d (%s), returning resources to root\n", instance->id, instance->name); + ret = mk_instance_release_resources(instance); + if (WARN_ON_ONCE(ret)) { + pr_crit("Retaining multikernel instance %d (%s) after resource release failed: %d\n", + instance->id, instance->name, ret); + return; + } mk_ipi_endpoint_unregister(instance); - mk_instance_return_all_cpus(instance); - mk_instance_return_pci_devices(instance); - mk_instance_return_platform_devices(instance); - mk_instance_free_memory(instance); - mk_instance_track_dump(instance, instance->state, MK_STATE_READY); if (instance->halt_data) memunmap(instance->halt_data); @@ -241,6 +263,7 @@ struct mk_instance *mk_instance_alloc(int id, const char *name) INIT_LIST_HEAD(&instance->list); INIT_LIST_HEAD(&instance->pci_devices); INIT_LIST_HEAD(&instance->platform_devices); + mk_pci_lease_instance_init(instance); kref_init(&instance->refcount); return instance; @@ -649,11 +672,6 @@ static int mk_instance_transfer_pci_devices(struct mk_instance *instance, const struct list_head *requested_devices, int requested_count) { - struct mk_pci_device *req_dev, *self_dev, *tmp; - int transferred = 0; - int not_found = 0; - bool found; - if (!mk_self || !mk_self->pci_devices_valid) { pr_err("No self instance or PCI devices not initialized\n"); return -EINVAL; @@ -666,90 +684,12 @@ static int mk_instance_transfer_pci_devices(struct mk_instance *instance, return 0; } - /* A spawn lends out of its tree */ - if (mk_manifest_phys()) { - list_for_each_entry(req_dev, requested_devices, list) { - u8 devfn = PCI_DEVFN(req_dev->slot, req_dev->func); - - if (!mk_of_pci_available(req_dev->domain, req_dev->bus, devfn)) { - pr_err("PCI device %04x:%02x:%02x.%x is not this kernel's to lend\n", - req_dev->domain, req_dev->bus, req_dev->slot, - req_dev->func); - not_found++; - } - } - if (not_found) - return -ENOENT; - list_for_each_entry(req_dev, requested_devices, list) { - int ret = mk_of_pci_lend(instance, req_dev->domain, req_dev->bus, - PCI_DEVFN(req_dev->slot, req_dev->func)); - - if (ret) - return ret; - transferred++; - } - instance->pci_devices_valid = true; - pr_info("Lent %d PCI devices to instance %d (%s)\n", - transferred, instance->id, instance->name); - return 0; - } - - list_for_each_entry(req_dev, requested_devices, list) { - found = false; - list_for_each_entry(self_dev, &mk_self->pci_devices, list) { - if (self_dev->vendor == req_dev->vendor && - self_dev->device == req_dev->device && - self_dev->domain == req_dev->domain && - self_dev->bus == req_dev->bus && - self_dev->slot == req_dev->slot && - self_dev->func == req_dev->func) { - found = true; - break; - } - } - if (!found) { - pr_err("PCI device %04x:%04x@%04x:%02x:%02x.%x not available in root pool\n", - req_dev->vendor, req_dev->device, req_dev->domain, - req_dev->bus, req_dev->slot, req_dev->func); - not_found++; - } - } - - if (not_found > 0) { - pr_err("Instance %d (%s): %d PCI devices not available\n", - instance->id, instance->name, not_found); - return -ENOENT; - } - - list_for_each_entry(req_dev, requested_devices, list) { - list_for_each_entry_safe(self_dev, tmp, &mk_self->pci_devices, list) { - if (self_dev->vendor == req_dev->vendor && - self_dev->device == req_dev->device && - self_dev->domain == req_dev->domain && - self_dev->bus == req_dev->bus && - self_dev->slot == req_dev->slot && - self_dev->func == req_dev->func) { - - list_del(&self_dev->list); - list_add_tail(&self_dev->list, &instance->pci_devices); - mk_self->pci_device_count--; - instance->pci_device_count++; - transferred++; - - pr_debug("Transferred PCI device %04x:%04x@%04x:%02x:%02x.%x to instance %d\n", - self_dev->vendor, self_dev->device, self_dev->domain, - self_dev->bus, self_dev->slot, self_dev->func, - instance->id); - break; - } - } - } - - instance->pci_devices_valid = true; - pr_info("Transferred %d PCI devices from root to instance %d (%s), root pool remaining: %d devices\n", - transferred, instance->id, instance->name, mk_self->pci_device_count); + /* Nested kernels cannot establish the host-owned VF lifecycle. */ + if (mk_manifest_phys()) + return -EOPNOTSUPP; - return 0; + return mk_pci_assign_devices(instance, requested_devices, + requested_count); } static int mk_instance_reserve_pci_devices(struct mk_instance *instance, @@ -864,49 +804,9 @@ static int mk_instance_reserve_platform_devices(struct mk_instance *instance, int mk_instance_add_pci_device(struct mk_instance *instance, u16 domain, u8 bus, u8 devfn) { - struct mk_pci_device *self_dev, *tmp; - u8 slot = PCI_SLOT(devfn); - u8 func = PCI_FUNC(devfn); - - if (!mk_self || !mk_self->pci_devices_valid) { - pr_err("No self instance or PCI devices not initialized\n"); - return -EINVAL; - } - - if (mk_manifest_phys()) { - int ret = mk_of_pci_lend(instance, domain, bus, devfn); - - if (ret) - pr_err("PCI device %04x:%02x:%02x.%x is not this kernel's to lend\n", - domain, bus, slot, func); - else - pr_info("Lent PCI device %04x:%02x:%02x.%x to instance %d\n", - domain, bus, slot, func, instance->id); - return ret; - } - - list_for_each_entry_safe(self_dev, tmp, &mk_self->pci_devices, list) { - if (self_dev->domain == domain && - self_dev->bus == bus && - self_dev->slot == slot && - self_dev->func == func) { - - list_del(&self_dev->list); - list_add_tail(&self_dev->list, &instance->pci_devices); - mk_self->pci_device_count--; - instance->pci_device_count++; - instance->pci_devices_valid = true; - - pr_info("Transferred PCI device %04x:%04x@%04x:%02x:%02x.%x to instance %d\n", - self_dev->vendor, self_dev->device, domain, bus, slot, func, - instance->id); - return 0; - } - } - - pr_err("PCI device %04x:%02x:%02x.%x not found in root pool\n", - domain, bus, slot, func); - return -ENOENT; + if (mk_manifest_phys()) + return -EOPNOTSUPP; + return mk_pci_assign_device(instance, domain, bus, devfn); } /** @@ -924,62 +824,9 @@ int mk_instance_add_pci_device(struct mk_instance *instance, int mk_instance_remove_pci_device(struct mk_instance *instance, u16 domain, u8 bus, u8 devfn) { - struct mk_pci_device *inst_dev, *tmp; - struct mk_pci_device *self_dev; - u8 slot = PCI_SLOT(devfn); - u8 func = PCI_FUNC(devfn); - - if (!instance->pci_devices_valid) { - pr_err("Instance %d PCI devices not initialized\n", instance->id); - return -EINVAL; - } - - if (!mk_self) { - pr_err("Cannot return PCI device: no self instance\n"); - return -EINVAL; - } - - list_for_each_entry_safe(inst_dev, tmp, &instance->pci_devices, list) { - if (inst_dev->domain == domain && - inst_dev->bus == bus && - inst_dev->slot == slot && - inst_dev->func == func) { - - if (mk_manifest_phys()) { - mk_of_pci_take_back(domain, bus, devfn); - list_del(&inst_dev->list); - kfree(inst_dev); - instance->pci_device_count--; - pr_info("Took PCI device %04x:%02x:%02x.%x back from instance %d\n", - domain, bus, slot, func, instance->id); - return 0; - } - - self_dev = kzalloc_obj(*self_dev, GFP_KERNEL); - if (!self_dev) - return -ENOMEM; - - *self_dev = *inst_dev; - INIT_LIST_HEAD(&self_dev->list); - - list_add_tail(&self_dev->list, &mk_self->pci_devices); - mk_self->pci_device_count++; - mk_self->pci_devices_valid = true; - - list_del(&inst_dev->list); - kfree(inst_dev); - instance->pci_device_count--; - - pr_info("Returned PCI device %04x:%04x@%04x:%02x:%02x.%x from instance %d to root\n", - self_dev->vendor, self_dev->device, domain, bus, slot, func, - instance->id); - return 0; - } - } - - pr_err("PCI device %04x:%02x:%02x.%x not found in instance %d\n", - domain, bus, slot, func, instance->id); - return -ENOENT; + if (mk_manifest_phys()) + return -EOPNOTSUPP; + return mk_pci_unassign_device(instance, domain, bus, devfn); } /** @@ -2003,9 +1850,14 @@ static int __init multikernel_init(void) { int ret; + ret = mk_pci_lease_system_init(); + if (ret) + return ret; + ret = mk_messaging_init(); if (ret < 0) { pr_err("Failed to initialize multikernel messaging: %d\n", ret); + mk_pci_lease_system_cleanup(); return ret; } @@ -2013,6 +1865,7 @@ static int __init multikernel_init(void) if (ret < 0) { pr_err("Failed to register system message handler: %d\n", ret); mk_messaging_cleanup(); + mk_pci_lease_system_cleanup(); return ret; } @@ -2021,6 +1874,7 @@ static int __init multikernel_init(void) pr_err("Failed to initialize multikernel hotplug: %d\n", ret); mk_unregister_msg_handler(MK_MSG_SYSTEM, mk_system_msg_handler); mk_messaging_cleanup(); + mk_pci_lease_system_cleanup(); return ret; } @@ -2030,6 +1884,7 @@ static int __init multikernel_init(void) mk_hotplug_cleanup(); mk_unregister_msg_handler(MK_MSG_SYSTEM, mk_system_msg_handler); mk_messaging_cleanup(); + mk_pci_lease_system_cleanup(); return ret; } diff --git a/kernel/multikernel/internal.h b/kernel/multikernel/internal.h index ba086d95ca2942..83806c4c565547 100644 --- a/kernel/multikernel/internal.h +++ b/kernel/multikernel/internal.h @@ -45,6 +45,20 @@ int mk_dt_generate_instance_dtb(struct mk_instance *instance, void **out_dtb, size_t *out_size); int mk_pci_parse_bdf(const char *pci_id, int len, u16 *domain, u8 *bus, u8 *slot, u8 *func); + +/* pci.c */ +int mk_pci_lease_system_init(void); +void mk_pci_lease_system_cleanup(void); +void mk_pci_lease_instance_init(struct mk_instance *instance); +int mk_pci_assign_devices(struct mk_instance *instance, + const struct list_head *requested_devices, + int requested_count); +int mk_pci_assign_device(struct mk_instance *instance, u16 domain, u8 bus, + u8 devfn); +int mk_pci_unassign_device(struct mk_instance *instance, u16 domain, u8 bus, + u8 devfn); +int mk_pci_release_assignments(struct mk_instance *instance); +int mk_instance_force_halt(struct mk_instance *instance); /* overlay.c */ extern struct kernfs_node *mk_overlay_root_kn; extern struct mutex mk_overlay_mutex; diff --git a/kernel/multikernel/kernfs.c b/kernel/multikernel/kernfs.c index c8fa6270e1fe6b..1698f6c4b2f5b5 100644 --- a/kernel/multikernel/kernfs.c +++ b/kernel/multikernel/kernfs.c @@ -407,6 +407,8 @@ static int mk_kernfs_rmdir(struct kernfs_node *kn) */ int mk_instance_destroy(struct mk_instance *instance) { + int ret; + lockdep_assert_held(&mk_instance_mutex); if (!instance) { @@ -426,6 +428,13 @@ int mk_instance_destroy(struct mk_instance *instance) return -EBUSY; } + ret = mk_instance_release_resources(instance); + if (ret) { + pr_err("Cannot remove instance '%s' (ID: %d): resource release failed: %d\n", + instance->name, instance->id, ret); + return ret; + } + list_del(&instance->list); idr_remove(&mk_instance_idr, instance->id); if (instance->kn) { diff --git a/kernel/multikernel/pci.c b/kernel/multikernel/pci.c index 23b713d9aeedb9..d94d20e7c1e42e 100644 --- a/kernel/multikernel/pci.c +++ b/kernel/multikernel/pci.c @@ -7,31 +7,579 @@ * the current instance. */ +#include +#include #include #include +#include +#include #include "internal.h" +struct mk_pci_assignment { + struct list_head instance_node; + struct list_head active_node; + struct list_head transaction_node; + struct mk_instance *instance; + struct mk_pci_device *inventory; + struct pci_dev *vf; + struct pci_dev *pf; + const struct device_driver *host_driver; + struct work_struct failure_work; + atomic_t failure_pending; + bool assigned; + bool inventory_moved; + bool expected_unbind; +}; + +static DEFINE_MUTEX(mk_pci_lease_mutex); +static DEFINE_SPINLOCK(mk_pci_active_lock); +static LIST_HEAD(mk_pci_active_assignments); +static bool mk_pci_notifier_registered; + +static bool mk_pci_device_live(struct pci_dev *pdev) +{ + return device_is_registered(&pdev->dev) && + !pci_dev_is_disconnected(pdev) && + pci_device_is_present(pdev); +} + +static bool mk_pci_device_matches_bdf(const struct mk_pci_device *device, + u16 domain, u8 bus, u8 devfn) +{ + return device->domain == domain && device->bus == bus && + device->slot == PCI_SLOT(devfn) && + device->func == PCI_FUNC(devfn); +} + static struct mk_pci_device * -mk_pci_find_assigned_bdf(u16 domain, u8 bus, u8 devfn) +mk_pci_find_device_bdf(struct list_head *devices, u16 domain, u8 bus, u8 devfn) { struct mk_pci_device *device; - u8 slot = PCI_SLOT(devfn); - u8 func = PCI_FUNC(devfn); - if (!root_instance || !root_instance->dtb_data || - !root_instance->pci_devices_valid) - return NULL; + list_for_each_entry(device, devices, list) { + if (mk_pci_device_matches_bdf(device, domain, bus, devfn)) + return device; + } + + return NULL; +} + +static bool mk_pci_inventory_matches(const struct mk_pci_device *left, + const struct mk_pci_device *right) +{ + return left->vendor == right->vendor && + left->device == right->device && + mk_pci_device_matches_bdf(left, right->domain, right->bus, + PCI_DEVFN(right->slot, right->func)); +} + +static struct mk_pci_device * +mk_pci_find_root_inventory(const struct mk_pci_device *requested) +{ + struct mk_pci_device *device; list_for_each_entry(device, &root_instance->pci_devices, list) { - if (device->domain == domain && device->bus == bus && - device->slot == slot && device->func == func) + if (mk_pci_inventory_matches(device, requested)) return device; } return NULL; } +static struct mk_pci_device * +mk_pci_find_root_bdf(u16 domain, u8 bus, u8 devfn) +{ + return mk_pci_find_device_bdf(&root_instance->pci_devices, domain, bus, + devfn); +} + +static struct mk_pci_assignment * +mk_pci_find_assignment(struct mk_instance *instance, u16 domain, u8 bus, + u8 devfn) +{ + struct mk_pci_assignment *assignment; + + list_for_each_entry(assignment, &instance->pci_assignments, + instance_node) { + if (mk_pci_device_matches_bdf(assignment->inventory, domain, bus, + devfn)) + return assignment; + } + + return NULL; +} + +static void mk_pci_assignment_failure_work(struct work_struct *work) +{ + struct mk_pci_assignment *assignment = + container_of(work, struct mk_pci_assignment, failure_work); + struct mk_instance *instance = assignment->instance; + int ret; + + pr_err("PCI assignment lease for %s was lost by instance %d (%s)\n", + pci_name(assignment->vf), instance->id, instance->name); + + if (READ_ONCE(instance->state) == MK_STATE_ACTIVE) { + ret = mk_instance_force_halt(instance); + if (ret) + pr_err("Failed to force halt instance %d after PCI lease loss: %d\n", + instance->id, ret); + } + + mk_instance_set_state(instance, MK_STATE_FAILED); +} + +static void mk_pci_schedule_failure(struct mk_pci_assignment *assignment) +{ + if (atomic_cmpxchg(&assignment->failure_pending, 0, 1)) + return; + + if (!schedule_work(&assignment->failure_work)) + atomic_set(&assignment->failure_pending, 0); +} + +static int mk_pci_bus_notify(struct notifier_block *nb, unsigned long action, + void *data) +{ + struct pci_dev *pdev = to_pci_dev(data); + struct mk_pci_assignment *assignment; + unsigned long flags; + + if (action != BUS_NOTIFY_DEL_DEVICE && + action != BUS_NOTIFY_REMOVED_DEVICE && + action != BUS_NOTIFY_UNBOUND_DRIVER) + return NOTIFY_DONE; + + spin_lock_irqsave(&mk_pci_active_lock, flags); + list_for_each_entry(assignment, &mk_pci_active_assignments, + active_node) { + if (pdev != assignment->vf && pdev != assignment->pf) + continue; + if (pdev == assignment->vf && + action == BUS_NOTIFY_UNBOUND_DRIVER && + assignment->expected_unbind) + continue; + mk_pci_schedule_failure(assignment); + } + spin_unlock_irqrestore(&mk_pci_active_lock, flags); + + return NOTIFY_OK; +} + +static struct notifier_block mk_pci_bus_notifier = { + .notifier_call = mk_pci_bus_notify, +}; + +static int +mk_pci_prepare_assignment(struct mk_instance *instance, + const struct mk_pci_device *requested, + struct list_head *transaction) +{ + struct mk_pci_assignment *assignment; + struct mk_pci_device *inventory; + struct pci_dev *vf; + struct pci_dev *pf; + struct pci_dev *physfn; + + inventory = mk_pci_find_root_inventory(requested); + if (!inventory) { + pr_err("PCI device %04x:%04x@%04x:%02x:%02x.%x is not available in the root pool\n", + requested->vendor, requested->device, requested->domain, + requested->bus, requested->slot, requested->func); + return -ENOENT; + } + + vf = pci_get_domain_bus_and_slot(inventory->domain, + inventory->bus, + PCI_DEVFN(inventory->slot, + inventory->func)); + if (!vf) + return -ENODEV; + + if (vf->vendor != inventory->vendor || + vf->device != inventory->device) { + pr_err("PCI identity changed for %s: expected %04x:%04x, found %04x:%04x\n", + pci_name(vf), inventory->vendor, inventory->device, + vf->vendor, vf->device); + pci_dev_put(vf); + return -ENODEV; + } + + physfn = pci_physfn(vf); + if (!vf->is_virtfn || physfn == vf) { + pr_err("PCI assignment only supports SR-IOV VFs, rejecting %s\n", + pci_name(vf)); + pci_dev_put(vf); + return -EOPNOTSUPP; + } + + if (!mk_pci_device_live(vf) || !mk_pci_device_live(physfn)) { + pci_dev_put(vf); + return -ENODEV; + } + + if (pci_is_dev_assigned(vf) || + mk_pci_find_assignment(instance, inventory->domain, + inventory->bus, + PCI_DEVFN(inventory->slot, + inventory->func))) { + pci_dev_put(vf); + return -EBUSY; + } + + pf = pci_dev_get(physfn); + assignment = kzalloc_obj(*assignment, GFP_KERNEL); + if (!assignment) { + pci_dev_put(pf); + pci_dev_put(vf); + return -ENOMEM; + } + + assignment->instance = instance; + assignment->inventory = inventory; + assignment->vf = vf; + assignment->pf = pf; + assignment->host_driver = vf->dev.driver; + if (assignment->host_driver && assignment->host_driver->owner && + !try_module_get(assignment->host_driver->owner)) { + kfree(assignment); + pci_dev_put(pf); + pci_dev_put(vf); + return -ENODEV; + } + + INIT_LIST_HEAD(&assignment->instance_node); + INIT_LIST_HEAD(&assignment->active_node); + INIT_LIST_HEAD(&assignment->transaction_node); + INIT_WORK(&assignment->failure_work, mk_pci_assignment_failure_work); + atomic_set(&assignment->failure_pending, 0); + list_add_tail(&assignment->instance_node, &instance->pci_assignments); + list_add_tail(&assignment->transaction_node, transaction); + + return 0; +} + +static int mk_pci_commit_assignment(struct mk_pci_assignment *assignment) +{ + struct pci_dev *vf = assignment->vf; + unsigned long flags; + int i; + + if (!mk_pci_device_live(vf) || !mk_pci_device_live(assignment->pf)) + return -ENODEV; + + if (vf->dev.driver != assignment->host_driver || + pci_is_dev_assigned(vf)) + return -EBUSY; + + pci_set_dev_assigned(vf); + assignment->assigned = true; + + spin_lock_irqsave(&mk_pci_active_lock, flags); + list_add_tail(&assignment->active_node, &mk_pci_active_assignments); + assignment->expected_unbind = true; + spin_unlock_irqrestore(&mk_pci_active_lock, flags); + + if (assignment->host_driver) + device_release_driver(&vf->dev); + + spin_lock_irqsave(&mk_pci_active_lock, flags); + assignment->expected_unbind = false; + spin_unlock_irqrestore(&mk_pci_active_lock, flags); + + if (vf->dev.driver) + return -EBUSY; + + for (i = 0; i < MK_PCI_RESOURCE_COUNT; i++) { + assignment->inventory->resources[i].start = + vf->resource[i].start; + assignment->inventory->resources[i].end = + vf->resource[i].end; + assignment->inventory->resources[i].flags = + vf->resource[i].flags; + } + assignment->inventory->resources_valid = true; + + list_move_tail(&assignment->inventory->list, + &assignment->instance->pci_devices); + root_instance->pci_device_count--; + assignment->instance->pci_device_count++; + assignment->instance->pci_devices_valid = true; + assignment->inventory_moved = true; + + pr_info("Leased SR-IOV VF %s to instance %d (%s)\n", + pci_name(vf), assignment->instance->id, + assignment->instance->name); + return 0; +} + +static int mk_pci_release_assignment(struct mk_pci_assignment *assignment) +{ + struct mk_instance *instance = assignment->instance; + struct pci_dev *vf = assignment->vf; + unsigned long flags; + int ret = 0; + + spin_lock_irqsave(&mk_pci_active_lock, flags); + if (!list_empty(&assignment->active_node)) + list_del_init(&assignment->active_node); + assignment->expected_unbind = false; + spin_unlock_irqrestore(&mk_pci_active_lock, flags); + + cancel_work_sync(&assignment->failure_work); + + if (assignment->assigned) { + pci_clear_dev_assigned(vf); + assignment->assigned = false; + } + + if (assignment->host_driver && mk_pci_device_live(vf)) { + if (!vf->dev.driver) { + ret = device_driver_attach(assignment->host_driver, + &vf->dev); + if (ret) + pr_err("Failed to restore driver %s to %s: %d\n", + assignment->host_driver->name, + pci_name(vf), ret); + } else if (vf->dev.driver != assignment->host_driver) { + pr_err("Cannot restore driver %s to %s: device is bound to %s\n", + assignment->host_driver->name, pci_name(vf), + vf->dev.driver->name); + ret = -EBUSY; + } + } + + if (assignment->inventory_moved && root_instance) { + list_move_tail(&assignment->inventory->list, + &root_instance->pci_devices); + instance->pci_device_count--; + root_instance->pci_device_count++; + root_instance->pci_devices_valid = true; + assignment->inventory_moved = false; + } + + if (assignment->host_driver && assignment->host_driver->owner) + module_put(assignment->host_driver->owner); + if (!list_empty(&assignment->transaction_node)) + list_del_init(&assignment->transaction_node); + list_del_init(&assignment->instance_node); + pci_dev_put(assignment->pf); + pci_dev_put(vf); + kfree(assignment); + + return ret; +} + +static void mk_pci_rollback_transaction(struct list_head *transaction) +{ + struct mk_pci_assignment *assignment; + + while (!list_empty(transaction)) { + assignment = list_last_entry(transaction, + struct mk_pci_assignment, + transaction_node); + mk_pci_release_assignment(assignment); + } +} + +static int mk_pci_commit_transaction(struct list_head *transaction) +{ + struct mk_pci_assignment *assignment; + int ret; + + list_for_each_entry(assignment, transaction, transaction_node) { + ret = mk_pci_commit_assignment(assignment); + if (ret) + return ret; + } + + while (!list_empty(transaction)) { + assignment = list_first_entry(transaction, + struct mk_pci_assignment, + transaction_node); + list_del_init(&assignment->transaction_node); + } + + return 0; +} + +void mk_pci_lease_instance_init(struct mk_instance *instance) +{ + INIT_LIST_HEAD(&instance->pci_assignments); +} + +int mk_pci_assign_devices(struct mk_instance *instance, + const struct list_head *requested_devices, + int requested_count) +{ + struct mk_pci_device *requested; + LIST_HEAD(transaction); + int prepared = 0; + int ret = 0; + + if (!instance || instance == root_instance || !requested_devices || + requested_count < 0) + return -EINVAL; + if (!root_instance || !root_instance->pci_devices_valid) + return -EINVAL; + + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + + list_for_each_entry(requested, requested_devices, list) { + ret = mk_pci_prepare_assignment(instance, requested, + &transaction); + if (ret) + goto rollback; + prepared++; + } + + if (prepared != requested_count) { + ret = -EINVAL; + goto rollback; + } + + ret = mk_pci_commit_transaction(&transaction); + if (ret) + goto rollback; + goto out; + +rollback: + mk_pci_rollback_transaction(&transaction); +out: + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + return ret; +} + +int mk_pci_assign_device(struct mk_instance *instance, u16 domain, u8 bus, + u8 devfn) +{ + struct mk_pci_device *inventory; + LIST_HEAD(transaction); + int ret; + + if (!instance || instance == root_instance) + return -EINVAL; + + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + + if (instance->state != MK_STATE_READY) { + ret = -EBUSY; + goto out; + } + if (!root_instance || !root_instance->pci_devices_valid) { + ret = -EINVAL; + goto out; + } + + inventory = mk_pci_find_root_bdf(domain, bus, devfn); + if (!inventory) { + ret = -ENOENT; + goto out; + } + + ret = mk_pci_prepare_assignment(instance, inventory, &transaction); + if (ret) + goto rollback; + ret = mk_pci_commit_transaction(&transaction); + if (ret) + goto rollback; + goto out; + +rollback: + mk_pci_rollback_transaction(&transaction); +out: + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + return ret; +} + +int mk_pci_unassign_device(struct mk_instance *instance, u16 domain, u8 bus, + u8 devfn) +{ + struct mk_pci_assignment *assignment; + int ret; + + if (!instance || instance == root_instance) + return -EINVAL; + + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + + if (instance->state != MK_STATE_READY) { + ret = -EBUSY; + goto out; + } + + assignment = mk_pci_find_assignment(instance, domain, bus, devfn); + if (!assignment) { + ret = -ENOENT; + goto out; + } + + ret = mk_pci_release_assignment(assignment); +out: + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + return ret; +} + +int mk_pci_release_assignments(struct mk_instance *instance) +{ + struct mk_pci_assignment *assignment; + int ret = 0; + int release_ret; + + if (!instance || instance == root_instance) + return 0; + + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + while (!list_empty(&instance->pci_assignments)) { + assignment = list_last_entry(&instance->pci_assignments, + struct mk_pci_assignment, + instance_node); + release_ret = mk_pci_release_assignment(assignment); + if (release_ret && !ret) + ret = release_ret; + } + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + return ret; +} + +int mk_pci_lease_system_init(void) +{ + int ret; + + ret = bus_register_notifier(&pci_bus_type, &mk_pci_bus_notifier); + if (!ret) + mk_pci_notifier_registered = true; + return ret; +} + +void mk_pci_lease_system_cleanup(void) +{ + if (!mk_pci_notifier_registered) + return; + bus_unregister_notifier(&pci_bus_type, &mk_pci_bus_notifier); + mk_pci_notifier_registered = false; +} + +static struct mk_pci_device * +mk_pci_find_assigned_bdf(u16 domain, u8 bus, u8 devfn) +{ + if (!root_instance || !root_instance->dtb_data || + !root_instance->pci_devices_valid) + return NULL; + + return mk_pci_find_device_bdf(&root_instance->pci_devices, + domain, bus, devfn); +} + static struct mk_pci_device *mk_pci_find_assigned(struct pci_bus *bus, int devfn) { return mk_pci_find_assigned_bdf(pci_domain_nr(bus), bus->number, devfn); From 327193c3d94a290442834ae14f2bad54c86a3a22 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Thu, 30 Jul 2026 08:21:22 +0300 Subject: [PATCH 06/24] multikernel: make resource reservation atomic Instance creation reserves memory, CPUs, PCI devices, host-bridge metadata, and platform devices. Returning an error after any one of those transfers can otherwise expose a partially populated instance and leak resources from the root. Validate configuration counts and lists before transfer, acquire each resource class in a fixed order, and unwind every completed step in reverse order. Centralize release so create failure, instance deletion, and final reference teardown share the same all-or-nothing semantics. Balance references acquired for remote memory add and remove operations on every success and error path so resource hotplug cannot pin a deleted instance. Signed-off-by: Nikolay Nikolaev --- include/linux/multikernel.h | 12 +- kernel/multikernel/core.c | 224 +++++++++++++++++++--------------- kernel/multikernel/internal.h | 1 + kernel/multikernel/kernfs.c | 2 +- kernel/multikernel/overlay.c | 18 ++- 5 files changed, 148 insertions(+), 109 deletions(-) diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index 685c6220c5187a..c5860d5ba78b54 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -896,14 +896,14 @@ struct mk_instance *mk_instance_get(struct mk_instance *instance); void __noreturn mk_halt_to_pool(void); /** - * mk_instance_reserve_resources() - Reserve CPU and memory resources for instance - * @instance: Instance to reserve resources for - * @config: Device tree configuration with memory size and CPU assignment + * mk_instance_reserve_resources() - Atomically reserve instance resources + * @instance: Empty instance to populate + * @config: Parsed memory, CPU, PCI, and platform resource configuration * - * Allocates the specified memory size from the multikernel pool, creates - * memory regions, and copies CPU assignment. + * Reserves every configured resource class or returns all resources acquired + * by the attempt. A failure never leaves a partially populated instance. * - * Returns 0 on success, negative error code on failure. + * Returns: 0 on success, negative error code on failure */ int mk_instance_reserve_resources(struct mk_instance *instance, const struct mk_dt_config *config); diff --git a/kernel/multikernel/core.c b/kernel/multikernel/core.c index 27f554a5827379..b8101e0f099594 100644 --- a/kernel/multikernel/core.c +++ b/kernel/multikernel/core.c @@ -97,13 +97,13 @@ static int mk_instance_return_pci_devices(struct mk_instance *instance) static void mk_instance_return_platform_devices(struct mk_instance *instance) { - struct mk_platform_device *plat_dev, *plat_tmp; - int returned_count = 0; + struct mk_platform_device *device, *tmp; + int returned = 0; - if (!instance || !instance->platform_devices_valid) + if (!instance || instance == mk_self || instance->id == 0) return; - - if (instance == mk_self || instance->id == 0) + if (!instance->platform_devices_valid && + list_empty(&instance->platform_devices)) return; if (!mk_self) { @@ -112,35 +112,21 @@ static void mk_instance_return_platform_devices(struct mk_instance *instance) goto cleanup; } - list_for_each_entry_safe(plat_dev, plat_tmp, &instance->platform_devices, list) { - struct mk_platform_device *self_dev; - - self_dev = kzalloc_obj(*self_dev, GFP_KERNEL); - if (!self_dev) - continue; - - *self_dev = *plat_dev; - INIT_LIST_HEAD(&self_dev->list); - - list_add_tail(&self_dev->list, &mk_self->platform_devices); + list_for_each_entry_safe(device, tmp, &instance->platform_devices, list) { + list_move_tail(&device->list, &mk_self->platform_devices); mk_self->platform_device_count++; mk_self->platform_devices_valid = true; - - pr_debug("Returned platform device '%s' from instance %d to root\n", - self_dev->name, instance->id); - - returned_count++; + returned++; } - if (returned_count > 0) { + if (returned) pr_info("Returned %d platform devices from instance %d (%s) to self instance\n", - returned_count, instance->id, instance->name); - } + returned, instance->id, instance->name); cleanup: - list_for_each_entry_safe(plat_dev, plat_tmp, &instance->platform_devices, list) { - list_del(&plat_dev->list); - kfree(plat_dev); + list_for_each_entry_safe(device, tmp, &instance->platform_devices, list) { + list_del(&device->list); + kfree(device); } instance->platform_device_count = 0; instance->platform_devices_valid = false; @@ -660,9 +646,9 @@ static int mk_instance_reserve_cpus(struct mk_instance *instance, const struct mk_dt_config *config) { if (!config->cpus) { - pr_warn("No CPU configuration for instance %d (%s)\n", - instance->id, instance->name); - return 0; + pr_err("No CPU configuration for instance %d (%s)\n", + instance->id, instance->name); + return -EINVAL; } return mk_instance_transfer_cpus(instance, config->cpus); @@ -695,13 +681,23 @@ static int mk_instance_transfer_pci_devices(struct mk_instance *instance, static int mk_instance_reserve_pci_devices(struct mk_instance *instance, const struct mk_dt_config *config) { - if (!config->pci_devices_valid || config->pci_device_count == 0) { + if (!config->pci_devices_valid) { + if (config->pci_device_count || !list_empty(&config->pci_devices)) + return -EINVAL; + instance->pci_devices_valid = true; + return 0; + } + if (!config->pci_device_count) { + if (!list_empty(&config->pci_devices)) + return -EINVAL; instance->pci_devices_valid = true; instance->pci_device_count = 0; pr_debug("No PCI devices to reserve for instance %d (%s)\n", instance->id, instance->name); return 0; } + if (list_empty(&config->pci_devices)) + return -EINVAL; return mk_instance_transfer_pci_devices(instance, &config->pci_devices, @@ -712,77 +708,87 @@ static int mk_instance_transfer_platform_devices(struct mk_instance *instance, const struct list_head *requested_devices, int requested_count) { - struct mk_platform_device *req_dev, *self_dev, *tmp; + struct mk_platform_device *requested, *other, *self_device; + int actual_count = 0; int transferred = 0; - int not_found = 0; - bool found; if (!mk_self || !mk_self->platform_devices_valid) { pr_err("No self instance or platform devices not initialized\n"); return -EINVAL; } - if (requested_count == 0 || list_empty(requested_devices)) { - pr_info("No platform devices requested for instance %d (%s)\n", - instance->id, instance->name); - instance->platform_devices_valid = true; - return 0; - } + if (requested_count <= 0 || list_empty(requested_devices)) + return -EINVAL; - list_for_each_entry(req_dev, requested_devices, list) { - found = false; - list_for_each_entry(self_dev, &mk_self->platform_devices, list) { - if (strcmp(self_dev->name, req_dev->name) == 0) { - found = true; + list_for_each_entry(requested, requested_devices, list) { + actual_count++; + list_for_each_entry(other, requested_devices, list) { + if (other == requested) break; - } + if (!strcmp(other->name, requested->name)) + return -EINVAL; } - if (!found) { - pr_err("Platform device '%s' not available in root pool\n", - req_dev->name); - not_found++; + self_device = NULL; + list_for_each_entry(other, &mk_self->platform_devices, list) { + if (!strcmp(other->name, requested->name)) { + self_device = other; + break; + } } + if (!self_device) + return -ENOENT; } + if (actual_count != requested_count) + return -EINVAL; - if (not_found > 0) { - pr_err("Instance %d (%s): %d platform devices not available\n", - instance->id, instance->name, not_found); - return -ENOENT; - } - - list_for_each_entry(req_dev, requested_devices, list) { - list_for_each_entry_safe(self_dev, tmp, &mk_self->platform_devices, list) { - if (strcmp(self_dev->name, req_dev->name) == 0) { - list_del(&self_dev->list); - list_add_tail(&self_dev->list, &instance->platform_devices); - mk_self->platform_device_count--; - instance->platform_device_count++; - transferred++; - - pr_debug("Transferred platform device '%s' to instance %d\n", - self_dev->name, instance->id); + list_for_each_entry(requested, requested_devices, list) { + self_device = NULL; + list_for_each_entry(other, &mk_self->platform_devices, list) { + if (!strcmp(other->name, requested->name)) { + self_device = other; break; } } + if (!self_device) + goto rollback; + list_move_tail(&self_device->list, &instance->platform_devices); + mk_self->platform_device_count--; + instance->platform_device_count++; + transferred++; } instance->platform_devices_valid = true; - pr_info("Transferred %d platform devices from root to instance %d (%s), root pool remaining: %d devices\n", - transferred, instance->id, instance->name, mk_self->platform_device_count); + pr_info("Transferred %d platform devices from self to instance %d (%s)\n", + transferred, instance->id, instance->name); return 0; + +rollback: + mk_instance_return_platform_devices(instance); + return -EIO; } static int mk_instance_reserve_platform_devices(struct mk_instance *instance, const struct mk_dt_config *config) { - if (!config->platform_devices_valid || config->platform_device_count == 0) { + if (!config->platform_devices_valid) { + if (config->platform_device_count || + !list_empty(&config->platform_devices)) + return -EINVAL; + instance->platform_devices_valid = true; + return 0; + } + if (!config->platform_device_count) { + if (!list_empty(&config->platform_devices)) + return -EINVAL; instance->platform_devices_valid = true; instance->platform_device_count = 0; pr_debug("No platform devices to reserve for instance %d (%s)\n", instance->id, instance->name); return 0; } + if (list_empty(&config->platform_devices)) + return -EINVAL; return mk_instance_transfer_platform_devices(instance, &config->platform_devices, @@ -1141,8 +1147,20 @@ void mk_instance_free_memory(struct mk_instance *instance) instance->id, instance->name); } +static bool mk_instance_resources_empty(const struct mk_instance *instance) +{ + return list_empty(&instance->memory_regions) && + !instance->instance_pool && !instance->region_count && + mk_cpu_set_empty(instance->cpus) && + list_empty(&instance->pci_devices) && + list_empty(&instance->pci_assignments) && + !instance->pci_device_count && + list_empty(&instance->platform_devices) && + !instance->platform_device_count; +} + /** - * mk_instance_reserve_resources() - Reserve memory and CPU resources for an instance + * mk_instance_reserve_resources() - Atomically reserve instance resources * @instance: Instance to reserve resources for * @config: Device tree configuration with memory regions and CPU assignment * @@ -1154,52 +1172,56 @@ void mk_instance_free_memory(struct mk_instance *instance) int mk_instance_reserve_resources(struct mk_instance *instance, const struct mk_dt_config *config) { + const char *failed_resource; + int release_ret; int ret; - if (!config || !instance) { + if (!config || !instance || !instance->cpus) { pr_err("Invalid parameters to mk_instance_reserve_resources\n"); return -EINVAL; } - - /* Free any existing memory regions first */ - mk_instance_free_memory(instance); + if (!mk_instance_resources_empty(instance)) + return -EBUSY; /* Reserve memory regions */ + failed_resource = "memory"; ret = mk_instance_reserve_memory(instance, config); - if (ret) { - pr_err("Failed to reserve memory regions for instance %d (%s): %d\n", - instance->id, instance->name, ret); - return ret; - } + if (ret) + goto rollback; /* Reserve CPU resources */ + failed_resource = "CPU"; ret = mk_instance_reserve_cpus(instance, config); - if (ret) { - pr_err("Failed to reserve CPU resources for instance %d (%s): %d\n", - instance->id, instance->name, ret); - /* Don't fail the whole operation for CPU reservation failure */ - pr_warn("Continuing without CPU assignment\n"); - } - - /* Reserve PCI device resources */ - ret = mk_instance_reserve_pci_devices(instance, config); - if (ret) { - pr_err("Failed to reserve PCI device resources for instance %d (%s): %d\n", - instance->id, instance->name, ret); - /* Don't fail the whole operation for PCI reservation failure */ - pr_warn("Continuing without PCI device assignment\n"); - } + if (ret) + goto rollback; /* Reserve platform device resources */ + failed_resource = "platform device"; ret = mk_instance_reserve_platform_devices(instance, config); - if (ret) { - pr_err("Failed to reserve platform device resources for instance %d (%s): %d\n", - instance->id, instance->name, ret); - /* Don't fail the whole operation for platform reservation failure */ - pr_warn("Continuing without platform device assignment\n"); - } + if (ret) + goto rollback; + + /* Commit fallible exclusive VF leases last. */ + failed_resource = "PCI device"; + ret = mk_instance_reserve_pci_devices(instance, config); + if (ret) + goto rollback; return 0; + +rollback: + pr_err("Failed to reserve %s resources for instance %d (%s): %d\n", + failed_resource, instance->id, instance->name, ret); + release_ret = mk_instance_release_resources(instance); + if (release_ret) { + mk_instance_set_state(instance, MK_STATE_FAILED); + return release_ret; + } + if (WARN_ON_ONCE(!mk_instance_resources_empty(instance))) { + mk_instance_set_state(instance, MK_STATE_FAILED); + return -EIO; + } + return ret; } /** diff --git a/kernel/multikernel/internal.h b/kernel/multikernel/internal.h index 83806c4c565547..f9552d29e5c82e 100644 --- a/kernel/multikernel/internal.h +++ b/kernel/multikernel/internal.h @@ -32,6 +32,7 @@ int mk_create_instance_from_dtb(const char *name, int id, const void *fdt, int chosen_node, int resources_node); struct mk_instance *mk_instance_find_by_name(const char *name); int mk_instance_destroy(struct mk_instance *instance); +void mk_instance_release_resources(struct mk_instance *instance); /* dts.c */ int mk_dt_parse_chosen(const void *fdt, int chosen_node, diff --git a/kernel/multikernel/kernfs.c b/kernel/multikernel/kernfs.c index 1698f6c4b2f5b5..b548ec023efccc 100644 --- a/kernel/multikernel/kernfs.c +++ b/kernel/multikernel/kernfs.c @@ -310,7 +310,7 @@ int mk_create_instance_from_dtb(const char *name, int id, const void *fdt, return 0; err_free_resources: - mk_instance_free_memory(instance); + mk_instance_release_resources(instance); err_free_idr: idr_remove(&mk_instance_idr, instance->id); err_remove_dir: diff --git a/kernel/multikernel/overlay.c b/kernel/multikernel/overlay.c index cf8d7ad82951e8..e718ee3ab7ddc3 100644 --- a/kernel/multikernel/overlay.c +++ b/kernel/multikernel/overlay.c @@ -1300,8 +1300,24 @@ static int mk_overlay_parse_and_apply(struct mk_overlay_tx *tx, const void *fdt) for (i = 0; i < nr; i++) { ret = mk_overlay_apply_fragment(tx, fdt, frags[i].node); - if (ret) + if (ret) { + int rollback_ret; + + /* + * A later fragment may depend on state created by an earlier + * one. Undo every completed fragment before reporting the + * transaction failure so the visible state stays atomic. + */ + while (--i >= 0) { + rollback_ret = + mk_overlay_rollback_fragment(tx, fdt, + frags[i].node); + if (rollback_ret) + pr_err("Overlay tx%d: failed to undo fragment@%x: %d\n", + tx->id, frags[i].unit, rollback_ret); + } break; + } } kfree(frags); From 69811302a1a185cc239e649d5367021d1843bfb0 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Thu, 30 Jul 2026 08:23:56 +0300 Subject: [PATCH 07/24] pci/multikernel: isolate VF DMA with a host IOMMU domain Exclusive VF ownership does not constrain DMA. Allocate a host-owned domain for each assignment and map only the memory owned by the instance. Update mappings with memory hotplug under the lease lifetime. Detach and destroy the domain before returning the VF to the host. Signed-off-by: Nikolay Nikolaev --- include/linux/multikernel.h | 3 + kernel/multikernel/Kconfig | 3 + kernel/multikernel/hotplug.c | 25 ++ kernel/multikernel/internal.h | 1 + kernel/multikernel/mem.c | 47 +++- kernel/multikernel/pci.c | 476 +++++++++++++++++++++++++++++++++- 6 files changed, 537 insertions(+), 18 deletions(-) diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index c5860d5ba78b54..eedcffb3198da7 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -676,6 +677,8 @@ struct mk_instance { int id; /* Kernel-assigned instance ID */ char *name; /* User-provided instance name */ enum mk_instance_state state; /* Current state */ + /* Serializes memory topology changes with PCI assignment leases. */ + struct mutex resource_mutex; /* Resource management - list of reserved memory regions */ struct list_head memory_regions; /* List of struct mk_memory_region */ diff --git a/kernel/multikernel/Kconfig b/kernel/multikernel/Kconfig index e7e242d0feac37..5b90f67d722b57 100644 --- a/kernel/multikernel/Kconfig +++ b/kernel/multikernel/Kconfig @@ -33,4 +33,7 @@ config MULTIKERNEL - A manifest handed to each spawn kernel on multikernel's own boot channel, describing its resources and message rings + SR-IOV VF assignment additionally requires PCI_IOV and an active + hardware IOMMU with isolated interrupt delivery. Assignment fails + closed when those isolation facilities are unavailable. If unsure, say N. diff --git a/kernel/multikernel/hotplug.c b/kernel/multikernel/hotplug.c index 1e5fdf743fd374..87a308faa7e0ec 100644 --- a/kernel/multikernel/hotplug.c +++ b/kernel/multikernel/hotplug.c @@ -1416,6 +1416,21 @@ int mk_send_cpu_add(int instance_id, mk_phys_cpu_t cpu_id, u32 numa_node, u32 fl return ret; } +static int mk_memory_change_allowed(struct mk_instance *instance) +{ + bool iommu_active; + + mutex_lock(&instance->resource_mutex); + iommu_active = mk_pci_iommu_lease_active_locked(instance); + mutex_unlock(&instance->resource_mutex); + if (!iommu_active) + return 0; + + pr_err("Cannot change memory for instance %d while an IOMMU lease is active\n", + instance->id); + return -EBUSY; +} + /** * mk_send_mem_add - Add memory to instance * @instance_id: Target instance ID @@ -1452,6 +1467,11 @@ int mk_send_mem_add(int instance_id, u64 start_pfn, u64 nr_pages, target_instance = mk_instance_find(instance_id); if (!target_instance) return -ENODEV; + ret = mk_memory_change_allowed(target_instance); + if (ret) { + mk_instance_put(target_instance); + return ret; + } /* For non-running instances, allocate memory from pool and add to instance */ if (target_instance->state != MK_STATE_ACTIVE) { @@ -1520,6 +1540,11 @@ int mk_send_mem_remove(int instance_id, u64 start_pfn, u64 nr_pages) target_instance = mk_instance_find(instance_id); if (!target_instance) return -ENODEV; + ret = mk_memory_change_allowed(target_instance); + if (ret) { + mk_instance_put(target_instance); + return ret; + } /* For non-running instances, just remove the memory region from the instance */ if (target_instance->state != MK_STATE_ACTIVE) { diff --git a/kernel/multikernel/internal.h b/kernel/multikernel/internal.h index f9552d29e5c82e..a30f53a4f50e7a 100644 --- a/kernel/multikernel/internal.h +++ b/kernel/multikernel/internal.h @@ -51,6 +51,7 @@ int mk_pci_parse_bdf(const char *pci_id, int len, u16 *domain, u8 *bus, int mk_pci_lease_system_init(void); void mk_pci_lease_system_cleanup(void); void mk_pci_lease_instance_init(struct mk_instance *instance); +bool mk_pci_iommu_lease_active_locked(struct mk_instance *instance); int mk_pci_assign_devices(struct mk_instance *instance, const struct list_head *requested_devices, int requested_count); diff --git a/kernel/multikernel/mem.c b/kernel/multikernel/mem.c index 2cd43fa2f5b239..22631d0462067f 100644 --- a/kernel/multikernel/mem.c +++ b/kernel/multikernel/mem.c @@ -437,17 +437,30 @@ int mk_instance_add_memory_region(struct mk_instance *instance, size_t size, phys_addr_t phys_addr; int ret; + if (!instance) + return -EINVAL; + + mutex_lock(&instance->resource_mutex); + if (mk_pci_iommu_lease_active_locked(instance)) { + pr_err("Cannot add memory to instance %d while an IOMMU lease is active\n", + instance->id); + ret = -EBUSY; + goto out_unlock; + } + phys_addr = multikernel_alloc(size, node); if (!phys_addr) { pr_err("Failed to allocate %zu bytes from multikernel pool for instance %d\n", size, instance->id); - return -ENOMEM; + ret = -ENOMEM; + goto out_unlock; } region = kzalloc(sizeof(*region), GFP_KERNEL); if (!region) { multikernel_free(phys_addr, size); - return -ENOMEM; + ret = -ENOMEM; + goto out_unlock; } region->res.name = kasprintf(GFP_KERNEL, "mk-instance-%d-%s-region-%d", @@ -455,7 +468,8 @@ int mk_instance_add_memory_region(struct mk_instance *instance, size_t size, if (!region->res.name) { kfree(region); multikernel_free(phys_addr, size); - return -ENOMEM; + ret = -ENOMEM; + goto out_unlock; } region->res.start = phys_addr; @@ -471,7 +485,7 @@ int mk_instance_add_memory_region(struct mk_instance *instance, size_t size, kfree(region->res.name); kfree(region); multikernel_free(phys_addr, size); - return ret; + goto out_unlock; } INIT_LIST_HEAD(®ion->list); @@ -482,7 +496,10 @@ int mk_instance_add_memory_region(struct mk_instance *instance, size_t size, (unsigned long long)phys_addr, (unsigned long long)(phys_addr + size - 1), size >> 20, instance->id, instance->name); - return 0; + ret = 0; +out_unlock: + mutex_unlock(&instance->resource_mutex); + return ret; } /* Does [phys_addr, phys_addr+size) back a segment of the loaded image? */ @@ -524,16 +541,26 @@ int mk_instance_remove_memory_region(struct mk_instance *instance, { struct mk_memory_region *region, *tmp; bool found = false; + int ret; if (!instance) return -EINVAL; + mutex_lock(&instance->resource_mutex); + if (mk_pci_iommu_lease_active_locked(instance)) { + pr_err("Cannot remove memory from instance %d while an IOMMU lease is active\n", + instance->id); + ret = -EBUSY; + goto out_unlock; + } + if (mk_range_backs_kimage(instance, phys_addr, size)) { pr_err("Refusing to remove 0x%llx-0x%llx from instance %d (%s): the loaded kernel image lives there\n", (unsigned long long)phys_addr, (unsigned long long)(phys_addr + size - 1), instance->id, instance->name); - return -EBUSY; + ret = -EBUSY; + goto out_unlock; } list_for_each_entry_safe(region, tmp, &instance->memory_regions, list) { @@ -563,10 +590,14 @@ int mk_instance_remove_memory_region(struct mk_instance *instance, (unsigned long long)phys_addr, (unsigned long long)(phys_addr + size - 1), instance->id, instance->name); - return -ENOENT; + ret = -ENOENT; + } else { + ret = 0; } - return 0; +out_unlock: + mutex_unlock(&instance->resource_mutex); + return ret; } /** diff --git a/kernel/multikernel/pci.c b/kernel/multikernel/pci.c index d94d20e7c1e42e..3fda9d87c6ef7b 100644 --- a/kernel/multikernel/pci.c +++ b/kernel/multikernel/pci.c @@ -8,8 +8,11 @@ */ #include +#include +#include #include #include +#include #include #include #include @@ -25,6 +28,14 @@ struct mk_pci_assignment { struct pci_dev *vf; struct pci_dev *pf; const struct device_driver *host_driver; + struct iommu_group *iommu_group; + struct iommu_domain *iommu_domain; + char *host_driver_override; + struct mutex iommu_mutex; /* Serializes IOMMU activation and teardown. */ + unsigned int iommu_mapped_regions; + bool iommu_dma_owner; + bool iommu_attached; + bool iommu_override_active; struct work_struct failure_work; atomic_t failure_pending; bool assigned; @@ -171,6 +182,400 @@ static struct notifier_block mk_pci_bus_notifier = { .notifier_call = mk_pci_bus_notify, }; +#if IS_ENABLED(CONFIG_IOMMU_API) +#define MK_PCI_ASSIGNMENT_DRIVER_NAME "multikernel-pci-assignment" + +static int mk_pci_iommu_assignment_probe(struct pci_dev *pdev, + const struct pci_device_id *id); +static void mk_pci_iommu_assignment_remove(struct pci_dev *pdev); + +static struct pci_driver mk_pci_assignment_driver = { + .name = MK_PCI_ASSIGNMENT_DRIVER_NAME, + .probe = mk_pci_iommu_assignment_probe, + .remove = mk_pci_iommu_assignment_remove, + .driver_managed_dma = true, +}; + +struct mk_pci_iommu_group_check { + struct device *vf; + unsigned int count; +}; + +static int mk_pci_iommu_check_group_device(struct device *dev, void *data) +{ + struct mk_pci_iommu_group_check *check = data; + + check->count++; + return dev == check->vf ? 0 : -EXDEV; +} + +static void mk_pci_iommu_free_resv_regions(struct list_head *regions) +{ + struct iommu_resv_region *region, *tmp; + + list_for_each_entry_safe(region, tmp, regions, list) { + list_del(®ion->list); + kfree(region); + } +} + +static int mk_pci_iommu_validate_group(struct mk_pci_assignment *assignment) +{ + struct mk_pci_iommu_group_check check = { + .vf = &assignment->vf->dev, + }; + struct iommu_resv_region *region; + LIST_HEAD(resv_regions); + int ret; + + ret = iommu_group_for_each_dev(assignment->iommu_group, &check, + mk_pci_iommu_check_group_device); + if (ret || check.count != 1) { + pr_err("IOMMU group %d for %s is not an isolated singleton group\n", + iommu_group_id(assignment->iommu_group), + pci_name(assignment->vf)); + return ret ?: -EXDEV; + } + + if (!iommu_group_has_isolated_msi(assignment->iommu_group)) { + pr_err("IOMMU group %d for %s lacks isolated MSI delivery\n", + iommu_group_id(assignment->iommu_group), + pci_name(assignment->vf)); + return -EPERM; + } + + ret = iommu_get_group_resv_regions(assignment->iommu_group, + &resv_regions); + if (ret) { + mk_pci_iommu_free_resv_regions(&resv_regions); + return ret; + } + + list_for_each_entry(region, &resv_regions, list) { + if (region->type != IOMMU_RESV_DIRECT && + region->type != IOMMU_RESV_DIRECT_RELAXABLE && + region->type != IOMMU_RESV_SW_MSI) + continue; + pr_err("IOMMU group %d for %s requires unsupported reserved region %#llx-%#llx type %u\n", + iommu_group_id(assignment->iommu_group), + pci_name(assignment->vf), + (unsigned long long)region->start, + (unsigned long long)(region->start + region->length - 1), + region->type); + ret = -EPERM; + break; + } + + mk_pci_iommu_free_resv_regions(&resv_regions); + return ret; +} + +static int +mk_pci_iommu_validate_region(struct mk_pci_assignment *assignment, + const struct mk_memory_region *region) +{ + struct iommu_domain *domain = assignment->iommu_domain; + resource_size_t start = region->res.start; + resource_size_t size = resource_size(®ion->res); + u64 dma_mask = dma_get_mask(&assignment->vf->dev); + u64 end; + unsigned long min_page_size; + + if (!size || check_add_overflow((u64)start, (u64)size - 1, &end)) + return -EOVERFLOW; + if (!domain->pgsize_bitmap) + return -EOPNOTSUPP; + + min_page_size = 1UL << __ffs(domain->pgsize_bitmap); + if (!IS_ALIGNED(start, min_page_size) || + !IS_ALIGNED(size, min_page_size)) { + pr_err("Instance %d memory %#llx-%#llx is not aligned to IOMMU page size %#lx\n", + assignment->instance->id, (unsigned long long)start, + (unsigned long long)end, min_page_size); + return -EINVAL; + } + if (start > ULONG_MAX || end > ULONG_MAX || end > dma_mask) { + pr_err("Instance %d memory %#llx-%#llx exceeds DMA addressability of %s\n", + assignment->instance->id, (unsigned long long)start, + (unsigned long long)end, pci_name(assignment->vf)); + return -ERANGE; + } + if (domain->geometry.force_aperture && + (start < domain->geometry.aperture_start || + end > domain->geometry.aperture_end)) { + pr_err("Instance %d memory %#llx-%#llx is outside the IOMMU aperture for %s\n", + assignment->instance->id, (unsigned long long)start, + (unsigned long long)end, pci_name(assignment->vf)); + return -ERANGE; + } + + return 0; +} + +static void mk_pci_iommu_unmap_regions(struct mk_pci_assignment *assignment) +{ + struct mk_memory_region *region; + unsigned int remaining = assignment->iommu_mapped_regions; + + list_for_each_entry(region, &assignment->instance->memory_regions, list) { + resource_size_t size; + size_t unmapped; + + if (!remaining) + break; + size = resource_size(®ion->res); + unmapped = iommu_unmap(assignment->iommu_domain, + region->res.start, size); + if (unmapped != size) + pr_err("IOMMU unmapped only %#zx of %#llx bytes for instance %d at %#llx\n", + unmapped, (unsigned long long)size, + assignment->instance->id, + (unsigned long long)region->res.start); + remaining--; + } + if (remaining) + pr_err("IOMMU lease for %s lost %u mapped instance regions\n", + pci_name(assignment->vf), remaining); + assignment->iommu_mapped_regions = 0; +} + +static void +__mk_pci_iommu_deactivate_assignment(struct mk_pci_assignment *assignment) +{ + if (assignment->iommu_attached) { + iommu_detach_group(assignment->iommu_domain, + assignment->iommu_group); + assignment->iommu_attached = false; + } + if (assignment->iommu_dma_owner) { + iommu_device_release_dma_owner(&assignment->vf->dev); + assignment->iommu_dma_owner = false; + } +} + +static void +mk_pci_iommu_deactivate_assignment(struct mk_pci_assignment *assignment) +{ + mutex_lock(&assignment->iommu_mutex); + __mk_pci_iommu_deactivate_assignment(assignment); + mutex_unlock(&assignment->iommu_mutex); +} + +static void mk_pci_iommu_release_assignment(struct mk_pci_assignment *assignment) +{ + mutex_lock(&assignment->iommu_mutex); + __mk_pci_iommu_deactivate_assignment(assignment); + + if (assignment->iommu_domain) { + mk_pci_iommu_unmap_regions(assignment); + iommu_domain_free(assignment->iommu_domain); + assignment->iommu_domain = NULL; + } + if (assignment->iommu_group) { + iommu_group_put(assignment->iommu_group); + assignment->iommu_group = NULL; + } + mutex_unlock(&assignment->iommu_mutex); +} + +static int mk_pci_iommu_prepare_assignment(struct mk_pci_assignment *assignment) +{ + struct mk_memory_region *region; + int ret; + + if (!device_iommu_mapped(&assignment->vf->dev)) { + pr_err("Cannot assign %s without an active hardware IOMMU\n", + pci_name(assignment->vf)); + return -EOPNOTSUPP; + } + if (!assignment->instance->region_count || + list_empty(&assignment->instance->memory_regions)) + return -EINVAL; + + assignment->iommu_group = iommu_group_get(&assignment->vf->dev); + if (!assignment->iommu_group) + return -ENODEV; + + ret = mk_pci_iommu_validate_group(assignment); + if (ret) + goto err_release; + + assignment->iommu_domain = + iommu_paging_domain_alloc(&assignment->vf->dev); + if (IS_ERR(assignment->iommu_domain)) { + ret = PTR_ERR(assignment->iommu_domain); + assignment->iommu_domain = NULL; + goto err_release; + } + + list_for_each_entry(region, &assignment->instance->memory_regions, list) { + resource_size_t size = resource_size(®ion->res); + + ret = mk_pci_iommu_validate_region(assignment, region); + if (ret) + goto err_release; + ret = iommu_map(assignment->iommu_domain, region->res.start, + region->res.start, size, IOMMU_READ | IOMMU_WRITE, + GFP_KERNEL); + if (ret) + goto err_release; + assignment->iommu_mapped_regions++; + } + + /* + * The domain blocks DMA outside these mappings, but translation-fault + * notification is not portable. In particular, Intel VT-d reports primary + * faults through dmar_fault() without invoking a legacy domain handler. + * Do not claim automatic instance failure on an IOMMU fault here. + */ + pr_info("Prepared host IOMMU domain for %s with %u instance regions\n", + pci_name(assignment->vf), assignment->iommu_mapped_regions); + return 0; + +err_release: + mk_pci_iommu_release_assignment(assignment); + return ret; +} + +static int mk_pci_iommu_commit_assignment(struct mk_pci_assignment *assignment) +{ + int ret; + + if (!assignment->iommu_domain) + return 0; + + if (assignment->vf->driver_override) { + assignment->host_driver_override = + kstrdup(assignment->vf->driver_override, GFP_KERNEL); + if (!assignment->host_driver_override) + return -ENOMEM; + } + + ret = driver_set_override(&assignment->vf->dev, + &assignment->vf->driver_override, + MK_PCI_ASSIGNMENT_DRIVER_NAME, + strlen(MK_PCI_ASSIGNMENT_DRIVER_NAME)); + if (ret) + return ret; + assignment->iommu_override_active = true; + pci_set_drvdata(assignment->vf, assignment); + ret = device_driver_attach(&mk_pci_assignment_driver.driver, + &assignment->vf->dev); + if (ret) + return ret; + if (assignment->vf->dev.driver != &mk_pci_assignment_driver.driver) + return -ENODEV; + return 0; +} + +static int mk_pci_iommu_assignment_probe(struct pci_dev *pdev, + const struct pci_device_id *id) +{ + struct mk_pci_assignment *assignment = pci_get_drvdata(pdev); + int ret; + + if (!assignment || assignment->vf != pdev || !assignment->iommu_domain) + return -ENODEV; + ret = iommu_device_claim_dma_owner(&assignment->vf->dev, assignment); + if (ret) + return ret; + assignment->iommu_dma_owner = true; + + ret = iommu_attach_group(assignment->iommu_domain, + assignment->iommu_group); + if (ret) + return ret; + assignment->iommu_attached = true; + pr_info("Attached %s to host-owned IOMMU domain for instance %d\n", + pci_name(assignment->vf), assignment->instance->id); + return 0; +} + +static void mk_pci_iommu_assignment_remove(struct pci_dev *pdev) +{ + struct mk_pci_assignment *assignment = pci_get_drvdata(pdev); + + if (assignment && assignment->vf == pdev) { + mk_pci_iommu_deactivate_assignment(assignment); + pci_set_drvdata(pdev, NULL); + } +} + +static int +mk_pci_iommu_restore_host_binding(struct mk_pci_assignment *assignment) +{ + struct pci_dev *vf = assignment->vf; + const char *override = assignment->host_driver_override ?: ""; + unsigned long flags; + int ret = 0; + + if (vf->dev.driver == &mk_pci_assignment_driver.driver) { + spin_lock_irqsave(&mk_pci_active_lock, flags); + assignment->expected_unbind = true; + spin_unlock_irqrestore(&mk_pci_active_lock, flags); + device_release_driver(&vf->dev); + spin_lock_irqsave(&mk_pci_active_lock, flags); + assignment->expected_unbind = false; + spin_unlock_irqrestore(&mk_pci_active_lock, flags); + } else if (vf->dev.driver) { + pr_err("Cannot release assignment driver from %s: device is bound to %s\n", + pci_name(vf), vf->dev.driver->name); + return -EBUSY; + } + + pci_set_drvdata(vf, NULL); + if (assignment->iommu_override_active) { + ret = driver_set_override(&vf->dev, &vf->driver_override, + override, strlen(override)); + if (ret) + return ret; + assignment->iommu_override_active = false; + } + return 0; +} + +static int mk_pci_iommu_system_init(void) +{ + return pci_register_driver(&mk_pci_assignment_driver); +} + +static void mk_pci_iommu_system_cleanup(void) +{ + pci_unregister_driver(&mk_pci_assignment_driver); +} +#else +static int mk_pci_iommu_prepare_assignment(struct mk_pci_assignment *assignment) +{ + pr_err("Cannot assign %s without CONFIG_IOMMU_API\n", + pci_name(assignment->vf)); + return -EOPNOTSUPP; +} + +static int mk_pci_iommu_commit_assignment(struct mk_pci_assignment *assignment) +{ + return 0; +} + +static void mk_pci_iommu_release_assignment(struct mk_pci_assignment *assignment) +{ +} + +static int +mk_pci_iommu_restore_host_binding(struct mk_pci_assignment *assignment) +{ + return 0; +} + +static int mk_pci_iommu_system_init(void) +{ + return 0; +} + +static void mk_pci_iommu_system_cleanup(void) +{ +} +#endif + static int mk_pci_prepare_assignment(struct mk_instance *instance, const struct mk_pci_device *requested, @@ -181,6 +586,7 @@ mk_pci_prepare_assignment(struct mk_instance *instance, struct pci_dev *vf; struct pci_dev *pf; struct pci_dev *physfn; + int ret; inventory = mk_pci_find_root_inventory(requested); if (!inventory) { @@ -252,18 +658,33 @@ mk_pci_prepare_assignment(struct mk_instance *instance, INIT_LIST_HEAD(&assignment->instance_node); INIT_LIST_HEAD(&assignment->active_node); INIT_LIST_HEAD(&assignment->transaction_node); + mutex_init(&assignment->iommu_mutex); INIT_WORK(&assignment->failure_work, mk_pci_assignment_failure_work); atomic_set(&assignment->failure_pending, 0); + + ret = mk_pci_iommu_prepare_assignment(assignment); + if (ret) + goto err_module; + list_add_tail(&assignment->instance_node, &instance->pci_assignments); list_add_tail(&assignment->transaction_node, transaction); return 0; + +err_module: + if (assignment->host_driver && assignment->host_driver->owner) + module_put(assignment->host_driver->owner); + kfree(assignment); + pci_dev_put(pf); + pci_dev_put(vf); + return ret; } static int mk_pci_commit_assignment(struct mk_pci_assignment *assignment) { struct pci_dev *vf = assignment->vf; unsigned long flags; + int ret; int i; if (!mk_pci_device_live(vf) || !mk_pci_device_live(assignment->pf)) @@ -291,6 +712,10 @@ static int mk_pci_commit_assignment(struct mk_pci_assignment *assignment) if (vf->dev.driver) return -EBUSY; + ret = mk_pci_iommu_commit_assignment(assignment); + if (ret) + return ret; + for (i = 0; i < MK_PCI_RESOURCE_COUNT; i++) { assignment->inventory->resources[i].start = vf->resource[i].start; @@ -319,6 +744,7 @@ static int mk_pci_release_assignment(struct mk_pci_assignment *assignment) struct mk_instance *instance = assignment->instance; struct pci_dev *vf = assignment->vf; unsigned long flags; + int binding_ret; int ret = 0; spin_lock_irqsave(&mk_pci_active_lock, flags); @@ -327,14 +753,18 @@ static int mk_pci_release_assignment(struct mk_pci_assignment *assignment) assignment->expected_unbind = false; spin_unlock_irqrestore(&mk_pci_active_lock, flags); - cancel_work_sync(&assignment->failure_work); - if (assignment->assigned) { pci_clear_dev_assigned(vf); assignment->assigned = false; } - if (assignment->host_driver && mk_pci_device_live(vf)) { + mk_pci_iommu_release_assignment(assignment); + cancel_work_sync(&assignment->failure_work); + binding_ret = mk_pci_iommu_restore_host_binding(assignment); + if (binding_ret) + ret = binding_ret; + + if (!binding_ret && assignment->host_driver && mk_pci_device_live(vf)) { if (!vf->dev.driver) { ret = device_driver_attach(assignment->host_driver, &vf->dev); @@ -359,6 +789,7 @@ static int mk_pci_release_assignment(struct mk_pci_assignment *assignment) assignment->inventory_moved = false; } + kfree(assignment->host_driver_override); if (assignment->host_driver && assignment->host_driver->owner) module_put(assignment->host_driver->owner); if (!list_empty(&assignment->transaction_node)) @@ -406,9 +837,19 @@ static int mk_pci_commit_transaction(struct list_head *transaction) void mk_pci_lease_instance_init(struct mk_instance *instance) { + mutex_init(&instance->resource_mutex); INIT_LIST_HEAD(&instance->pci_assignments); } +bool mk_pci_iommu_lease_active_locked(struct mk_instance *instance) +{ + if (!instance) + return false; + + lockdep_assert_held(&instance->resource_mutex); + return !list_empty(&instance->pci_assignments); +} + int mk_pci_assign_devices(struct mk_instance *instance, const struct list_head *requested_devices, int requested_count) @@ -424,6 +865,7 @@ int mk_pci_assign_devices(struct mk_instance *instance, if (!root_instance || !root_instance->pci_devices_valid) return -EINVAL; + mutex_lock(&instance->resource_mutex); mutex_lock(&mk_pci_lease_mutex); pci_lock_rescan_remove(); @@ -450,6 +892,7 @@ int mk_pci_assign_devices(struct mk_instance *instance, out: pci_unlock_rescan_remove(); mutex_unlock(&mk_pci_lease_mutex); + mutex_unlock(&instance->resource_mutex); return ret; } @@ -463,6 +906,7 @@ int mk_pci_assign_device(struct mk_instance *instance, u16 domain, u8 bus, if (!instance || instance == root_instance) return -EINVAL; + mutex_lock(&instance->resource_mutex); mutex_lock(&mk_pci_lease_mutex); pci_lock_rescan_remove(); @@ -494,6 +938,7 @@ int mk_pci_assign_device(struct mk_instance *instance, u16 domain, u8 bus, out: pci_unlock_rescan_remove(); mutex_unlock(&mk_pci_lease_mutex); + mutex_unlock(&instance->resource_mutex); return ret; } @@ -506,6 +951,7 @@ int mk_pci_unassign_device(struct mk_instance *instance, u16 domain, u8 bus, if (!instance || instance == root_instance) return -EINVAL; + mutex_lock(&instance->resource_mutex); mutex_lock(&mk_pci_lease_mutex); pci_lock_rescan_remove(); @@ -524,6 +970,7 @@ int mk_pci_unassign_device(struct mk_instance *instance, u16 domain, u8 bus, out: pci_unlock_rescan_remove(); mutex_unlock(&mk_pci_lease_mutex); + mutex_unlock(&instance->resource_mutex); return ret; } @@ -536,6 +983,7 @@ int mk_pci_release_assignments(struct mk_instance *instance) if (!instance || instance == root_instance) return 0; + mutex_lock(&instance->resource_mutex); mutex_lock(&mk_pci_lease_mutex); pci_lock_rescan_remove(); while (!list_empty(&instance->pci_assignments)) { @@ -548,6 +996,7 @@ int mk_pci_release_assignments(struct mk_instance *instance) } pci_unlock_rescan_remove(); mutex_unlock(&mk_pci_lease_mutex); + mutex_unlock(&instance->resource_mutex); return ret; } @@ -555,18 +1004,25 @@ int mk_pci_lease_system_init(void) { int ret; + ret = mk_pci_iommu_system_init(); + if (ret) + return ret; ret = bus_register_notifier(&pci_bus_type, &mk_pci_bus_notifier); - if (!ret) - mk_pci_notifier_registered = true; - return ret; + if (ret) { + mk_pci_iommu_system_cleanup(); + return ret; + } + mk_pci_notifier_registered = true; + return 0; } void mk_pci_lease_system_cleanup(void) { - if (!mk_pci_notifier_registered) - return; - bus_unregister_notifier(&pci_bus_type, &mk_pci_bus_notifier); - mk_pci_notifier_registered = false; + if (mk_pci_notifier_registered) { + bus_unregister_notifier(&pci_bus_type, &mk_pci_bus_notifier); + mk_pci_notifier_registered = false; + } + mk_pci_iommu_system_cleanup(); } static struct mk_pci_device * From 05f183bfb25ac02117bd7747631a7d35ad772f7a Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Thu, 30 Jul 2026 08:23:56 +0300 Subject: [PATCH 08/24] pci/multikernel: quiesce VFs before releasing leases Returning a VF while it can still issue DMA races the IOMMU teardown and host-driver reprobe. Unexpected PF or VF removal must also stop an active instance instead of silently losing its assigned device. Clear bus mastering, wait for pending transactions, and issue function-level reset while the assignment domain is still attached. Then detach and free the domain, restore the saved driver override and host driver, and only afterwards return the inventory to the root. Lease-loss notifications force an active instance to halt and mark it failed. Rollback entries that were prepared but never committed skip device quiesce and driver restoration, releasing only their prepared IOMMU resources. Signed-off-by: Nikolay Nikolaev --- kernel/multikernel/internal.h | 2 +- kernel/multikernel/kernfs.c | 17 ++- kernel/multikernel/pci.c | 199 ++++++++++++++++++++++++---------- 3 files changed, 159 insertions(+), 59 deletions(-) diff --git a/kernel/multikernel/internal.h b/kernel/multikernel/internal.h index a30f53a4f50e7a..137001c8172524 100644 --- a/kernel/multikernel/internal.h +++ b/kernel/multikernel/internal.h @@ -32,7 +32,7 @@ int mk_create_instance_from_dtb(const char *name, int id, const void *fdt, int chosen_node, int resources_node); struct mk_instance *mk_instance_find_by_name(const char *name); int mk_instance_destroy(struct mk_instance *instance); -void mk_instance_release_resources(struct mk_instance *instance); +int mk_instance_release_resources(struct mk_instance *instance); /* dts.c */ int mk_dt_parse_chosen(const void *fdt, int chosen_node, diff --git a/kernel/multikernel/kernfs.c b/kernel/multikernel/kernfs.c index b548ec023efccc..90170044f31a81 100644 --- a/kernel/multikernel/kernfs.c +++ b/kernel/multikernel/kernfs.c @@ -240,6 +240,7 @@ int mk_create_instance_from_dtb(const char *name, int id, const void *fdt, struct mk_instance *instance; struct kernfs_node *kn; struct mk_dt_config config; + int release_ret; int ret; int allocated_id; @@ -310,7 +311,16 @@ int mk_create_instance_from_dtb(const char *name, int id, const void *fdt, return 0; err_free_resources: - mk_instance_release_resources(instance); + release_ret = mk_instance_release_resources(instance); + if (release_ret) { + pr_crit("Retaining failed instance '%s' because PCI cleanup failed: %d\n", + name, release_ret); + list_add_tail(&instance->list, &mk_instance_list); + kernfs_activate(kn); + mk_instance_set_state(instance, MK_STATE_FAILED); + mk_dt_config_free(&config); + return release_ret; + } err_free_idr: idr_remove(&mk_instance_idr, instance->id); err_remove_dir: @@ -430,8 +440,9 @@ int mk_instance_destroy(struct mk_instance *instance) ret = mk_instance_release_resources(instance); if (ret) { - pr_err("Cannot remove instance '%s' (ID: %d): resource release failed: %d\n", - instance->name, instance->id, ret); + pr_crit("Cannot remove instance '%s' while PCI cleanup is unsafe: %d\n", + instance->name, ret); + mk_instance_set_state(instance, MK_STATE_FAILED); return ret; } diff --git a/kernel/multikernel/pci.c b/kernel/multikernel/pci.c index 3fda9d87c6ef7b..f394c6a8638a51 100644 --- a/kernel/multikernel/pci.c +++ b/kernel/multikernel/pci.c @@ -182,6 +182,20 @@ static struct notifier_block mk_pci_bus_notifier = { .notifier_call = mk_pci_bus_notify, }; +static void +mk_pci_release_bound_driver(struct mk_pci_assignment *assignment) +{ + unsigned long flags; + + spin_lock_irqsave(&mk_pci_active_lock, flags); + assignment->expected_unbind = true; + spin_unlock_irqrestore(&mk_pci_active_lock, flags); + device_release_driver(&assignment->vf->dev); + spin_lock_irqsave(&mk_pci_active_lock, flags); + assignment->expected_unbind = false; + spin_unlock_irqrestore(&mk_pci_active_lock, flags); +} + #if IS_ENABLED(CONFIG_IOMMU_API) #define MK_PCI_ASSIGNMENT_DRIVER_NAME "multikernel-pci-assignment" @@ -339,6 +353,43 @@ static void mk_pci_iommu_unmap_regions(struct mk_pci_assignment *assignment) assignment->iommu_mapped_regions = 0; } +static int +mk_pci_quiesce_assignment(struct mk_pci_assignment *assignment) +{ + struct pci_dev *vf = assignment->vf; + bool transactions_drained; + int ret; + + if (!mk_pci_device_live(vf)) + return 0; + + /* + * Releasing DMA ownership restores the group's default domain. Stop new + * DMA first, drain requests already issued, and reset the VF while the + * assignment domain still contains any stragglers. + */ + pci_clear_master(vf); + transactions_drained = pci_wait_for_pending_transaction(vf); + ret = pcie_reset_flr(vf, false); + if (!ret) + return 0; + if (ret == -ENOTTY && transactions_drained) + return 0; + + if (!transactions_drained) + pr_err("Timed out draining DMA from assigned VF %s\n", + pci_name(vf)); + if (ret != -ENOTTY) + pr_err("Failed to reset assigned VF %s: %d\n", + pci_name(vf), ret); + + /* + * Keep the assignment domain attached when the device cannot be made + * safe. The lease owner can retry teardown after the instance halts. + */ + return ret == -ENOTTY ? -ETIMEDOUT : ret; +} + static void __mk_pci_iommu_deactivate_assignment(struct mk_pci_assignment *assignment) { @@ -494,29 +545,35 @@ static int mk_pci_iommu_assignment_probe(struct pci_dev *pdev, static void mk_pci_iommu_assignment_remove(struct pci_dev *pdev) { struct mk_pci_assignment *assignment = pci_get_drvdata(pdev); + int ret; - if (assignment && assignment->vf == pdev) { - mk_pci_iommu_deactivate_assignment(assignment); + if (!assignment || assignment->vf != pdev) + return; + + if (READ_ONCE(assignment->expected_unbind)) { pci_set_drvdata(pdev, NULL); + return; } + + ret = mk_pci_quiesce_assignment(assignment); + if (ret) { + pr_crit("Keeping IOMMU containment for %s after unsafe driver removal: %d\n", + pci_name(pdev), ret); + mk_pci_schedule_failure(assignment); + } else { + mk_pci_iommu_deactivate_assignment(assignment); + } + pci_set_drvdata(pdev, NULL); } -static int -mk_pci_iommu_restore_host_binding(struct mk_pci_assignment *assignment) +static int mk_pci_restore_host_binding(struct mk_pci_assignment *assignment) { struct pci_dev *vf = assignment->vf; const char *override = assignment->host_driver_override ?: ""; - unsigned long flags; int ret = 0; if (vf->dev.driver == &mk_pci_assignment_driver.driver) { - spin_lock_irqsave(&mk_pci_active_lock, flags); - assignment->expected_unbind = true; - spin_unlock_irqrestore(&mk_pci_active_lock, flags); - device_release_driver(&vf->dev); - spin_lock_irqsave(&mk_pci_active_lock, flags); - assignment->expected_unbind = false; - spin_unlock_irqrestore(&mk_pci_active_lock, flags); + mk_pci_release_bound_driver(assignment); } else if (vf->dev.driver) { pr_err("Cannot release assignment driver from %s: device is bound to %s\n", pci_name(vf), vf->dev.driver->name); @@ -531,6 +588,24 @@ mk_pci_iommu_restore_host_binding(struct mk_pci_assignment *assignment) return ret; assignment->iommu_override_active = false; } + + mk_pci_iommu_deactivate_assignment(assignment); + if (assignment->host_driver && mk_pci_device_live(vf)) { + if (!vf->dev.driver) { + ret = device_driver_attach(assignment->host_driver, &vf->dev); + if (ret) { + pr_err("Failed to restore driver %s to %s: %d\n", + assignment->host_driver->name, + pci_name(vf), ret); + return ret; + } + } else if (vf->dev.driver != assignment->host_driver) { + pr_err("Cannot restore driver %s to %s: device is bound to %s\n", + assignment->host_driver->name, pci_name(vf), + vf->dev.driver->name); + return -EBUSY; + } + } return 0; } @@ -544,6 +619,17 @@ static void mk_pci_iommu_system_cleanup(void) pci_unregister_driver(&mk_pci_assignment_driver); } #else +static int +mk_pci_quiesce_assignment(struct mk_pci_assignment *assignment) +{ + return 0; +} + +static void +mk_pci_iommu_deactivate_assignment(struct mk_pci_assignment *assignment) +{ +} + static int mk_pci_iommu_prepare_assignment(struct mk_pci_assignment *assignment) { pr_err("Cannot assign %s without CONFIG_IOMMU_API\n", @@ -560,8 +646,7 @@ static void mk_pci_iommu_release_assignment(struct mk_pci_assignment *assignment { } -static int -mk_pci_iommu_restore_host_binding(struct mk_pci_assignment *assignment) +static int mk_pci_restore_host_binding(struct mk_pci_assignment *assignment) { return 0; } @@ -699,15 +784,10 @@ static int mk_pci_commit_assignment(struct mk_pci_assignment *assignment) spin_lock_irqsave(&mk_pci_active_lock, flags); list_add_tail(&assignment->active_node, &mk_pci_active_assignments); - assignment->expected_unbind = true; spin_unlock_irqrestore(&mk_pci_active_lock, flags); if (assignment->host_driver) - device_release_driver(&vf->dev); - - spin_lock_irqsave(&mk_pci_active_lock, flags); - assignment->expected_unbind = false; - spin_unlock_irqrestore(&mk_pci_active_lock, flags); + mk_pci_release_bound_driver(assignment); if (vf->dev.driver) return -EBUSY; @@ -744,8 +824,18 @@ static int mk_pci_release_assignment(struct mk_pci_assignment *assignment) struct mk_instance *instance = assignment->instance; struct pci_dev *vf = assignment->vf; unsigned long flags; - int binding_ret; - int ret = 0; + int ret; + + if (!assignment->assigned) + goto release_resources; + + ret = mk_pci_quiesce_assignment(assignment); + if (ret) + return ret; + + ret = mk_pci_restore_host_binding(assignment); + if (ret) + return ret; spin_lock_irqsave(&mk_pci_active_lock, flags); if (!list_empty(&assignment->active_node)) @@ -758,27 +848,9 @@ static int mk_pci_release_assignment(struct mk_pci_assignment *assignment) assignment->assigned = false; } +release_resources: mk_pci_iommu_release_assignment(assignment); cancel_work_sync(&assignment->failure_work); - binding_ret = mk_pci_iommu_restore_host_binding(assignment); - if (binding_ret) - ret = binding_ret; - - if (!binding_ret && assignment->host_driver && mk_pci_device_live(vf)) { - if (!vf->dev.driver) { - ret = device_driver_attach(assignment->host_driver, - &vf->dev); - if (ret) - pr_err("Failed to restore driver %s to %s: %d\n", - assignment->host_driver->name, - pci_name(vf), ret); - } else if (vf->dev.driver != assignment->host_driver) { - pr_err("Cannot restore driver %s to %s: device is bound to %s\n", - assignment->host_driver->name, pci_name(vf), - vf->dev.driver->name); - ret = -EBUSY; - } - } if (assignment->inventory_moved && root_instance) { list_move_tail(&assignment->inventory->list, @@ -799,19 +871,28 @@ static int mk_pci_release_assignment(struct mk_pci_assignment *assignment) pci_dev_put(vf); kfree(assignment); - return ret; + return 0; } -static void mk_pci_rollback_transaction(struct list_head *transaction) +static int mk_pci_rollback_transaction(struct list_head *transaction) { - struct mk_pci_assignment *assignment; + struct mk_pci_assignment *assignment, *tmp; + int rollback_ret = 0; + int ret; - while (!list_empty(transaction)) { - assignment = list_last_entry(transaction, - struct mk_pci_assignment, - transaction_node); - mk_pci_release_assignment(assignment); + list_for_each_entry_safe_reverse(assignment, tmp, transaction, + transaction_node) { + ret = mk_pci_release_assignment(assignment); + if (!ret) + continue; + pr_crit("Failed to roll back PCI assignment for %s: %d\n", + pci_name(assignment->vf), ret); + list_del_init(&assignment->transaction_node); + if (!rollback_ret) + rollback_ret = ret; } + + return rollback_ret; } static int mk_pci_commit_transaction(struct list_head *transaction) @@ -858,6 +939,7 @@ int mk_pci_assign_devices(struct mk_instance *instance, LIST_HEAD(transaction); int prepared = 0; int ret = 0; + int rollback_ret; if (!instance || instance == root_instance || !requested_devices || requested_count < 0) @@ -888,7 +970,9 @@ int mk_pci_assign_devices(struct mk_instance *instance, goto out; rollback: - mk_pci_rollback_transaction(&transaction); + rollback_ret = mk_pci_rollback_transaction(&transaction); + if (rollback_ret) + ret = rollback_ret; out: pci_unlock_rescan_remove(); mutex_unlock(&mk_pci_lease_mutex); @@ -902,6 +986,7 @@ int mk_pci_assign_device(struct mk_instance *instance, u16 domain, u8 bus, struct mk_pci_device *inventory; LIST_HEAD(transaction); int ret; + int rollback_ret; if (!instance || instance == root_instance) return -EINVAL; @@ -934,7 +1019,9 @@ int mk_pci_assign_device(struct mk_instance *instance, u16 domain, u8 bus, goto out; rollback: - mk_pci_rollback_transaction(&transaction); + rollback_ret = mk_pci_rollback_transaction(&transaction); + if (rollback_ret) + ret = rollback_ret; out: pci_unlock_rescan_remove(); mutex_unlock(&mk_pci_lease_mutex); @@ -978,7 +1065,6 @@ int mk_pci_release_assignments(struct mk_instance *instance) { struct mk_pci_assignment *assignment; int ret = 0; - int release_ret; if (!instance || instance == root_instance) return 0; @@ -990,9 +1076,12 @@ int mk_pci_release_assignments(struct mk_instance *instance) assignment = list_last_entry(&instance->pci_assignments, struct mk_pci_assignment, instance_node); - release_ret = mk_pci_release_assignment(assignment); - if (release_ret && !ret) - ret = release_ret; + ret = mk_pci_release_assignment(assignment); + if (ret) { + pr_crit("Instance %d retains unsafe PCI lease for %s: %d\n", + instance->id, pci_name(assignment->vf), ret); + break; + } } pci_unlock_rescan_remove(); mutex_unlock(&mk_pci_lease_mutex); From b2b989f98dd8d27478b145812ce11b1e096b0903 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Tue, 4 Aug 2026 00:50:57 +0300 Subject: [PATCH 09/24] pci/multikernel: reset assigned VFs before respawn A stopped or force-halted instance can leave bus mastering enabled and DMA in flight while its next boot rewrites the same memory. Lease teardown resets the VF, but a respawn retains the lease and previously skipped that protection. Require assigned VFs to support FLR. After confirming that all instance CPUs are parked, clear bus mastering, drain pending transactions, and reset every leased VF while its restrictive IOMMU domain remains attached. Abort the restart if any device cannot be made safe. Signed-off-by: Nikolay Nikolaev --- include/linux/multikernel.h | 1 + kernel/kexec_core.c | 12 +++++++ kernel/multikernel/pci.c | 62 +++++++++++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+) diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index eedcffb3198da7..bc83af950640c3 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -1001,6 +1001,7 @@ bool mk_manifest_rejected(void); /* Build the manifest for a spawn (host, kexec path) */ int mk_manifest_finalize(struct kimage *image); +int mk_pci_prepare_instance_start(struct mk_instance *instance); #else static inline bool multikernel_allow_emergency_restart(void) { diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index 709838b7dfe1da..7272f82400e8ed 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -1744,6 +1744,18 @@ int multikernel_kexec_by_id(int mk_id) goto unlock; } + /* + * Stop and reset every leased VF before rewriting instance memory. A + * force-halted kernel may have left bus mastering enabled and DMA in + * flight into the image that is about to be reused. + */ + rc = mk_pci_prepare_instance_start(instance); + if (rc) { + pr_err("Failed to prepare PCI assignments for instance %d restart: %d\n", + mk_id, rc); + goto unlock; + } + /* * Booting consumes the image: the spawn kernel writes its .data and * patches its own text, so the copy in instance memory is spent once diff --git a/kernel/multikernel/pci.c b/kernel/multikernel/pci.c index f394c6a8638a51..af68dce079c7c9 100644 --- a/kernel/multikernel/pci.c +++ b/kernel/multikernel/pci.c @@ -390,6 +390,38 @@ mk_pci_quiesce_assignment(struct mk_pci_assignment *assignment) return ret == -ENOTTY ? -ETIMEDOUT : ret; } +static int +mk_pci_reset_assignment_for_start(struct mk_pci_assignment *assignment) +{ + struct pci_dev *vf = assignment->vf; + int ret; + + if (!assignment->assigned || !assignment->iommu_attached) + return -EINVAL; + if (!mk_pci_device_live(vf)) + return -ENODEV; + + /* + * A stopped instance may have left DMA active. Keep its restrictive + * domain attached while stopping new requests, draining old ones, and + * resetting device state before the instance image is reused. + */ + pci_clear_master(vf); + if (!pci_wait_for_pending_transaction(vf)) { + pr_err("Timed out draining assigned VF %s before instance restart\n", + pci_name(vf)); + return -ETIMEDOUT; + } + + ret = pcie_reset_flr(vf, false); + if (ret) { + pr_err("Failed to reset assigned VF %s before instance restart: %d\n", + pci_name(vf), ret); + return ret == -ENOTTY ? -EOPNOTSUPP : ret; + } + return 0; +} + static void __mk_pci_iommu_deactivate_assignment(struct mk_pci_assignment *assignment) { @@ -709,6 +741,13 @@ mk_pci_prepare_assignment(struct mk_instance *instance, pci_dev_put(vf); return -ENODEV; } + ret = pcie_reset_flr(vf, true); + if (ret) { + pr_err("PCI assignment requires FLR for safe instance restart, rejecting %s\n", + pci_name(vf)); + pci_dev_put(vf); + return -EOPNOTSUPP; + } if (pci_is_dev_assigned(vf) || mk_pci_find_assignment(instance, inventory->domain, @@ -1089,6 +1128,29 @@ int mk_pci_release_assignments(struct mk_instance *instance) return ret; } +int mk_pci_prepare_instance_start(struct mk_instance *instance) +{ + struct mk_pci_assignment *assignment; + int ret = 0; + + if (!instance || instance == root_instance) + return -EINVAL; + + mutex_lock(&instance->resource_mutex); + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + list_for_each_entry(assignment, &instance->pci_assignments, + instance_node) { + ret = mk_pci_reset_assignment_for_start(assignment); + if (ret) + break; + } + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + mutex_unlock(&instance->resource_mutex); + return ret; +} + int mk_pci_lease_system_init(void) { int ret; From 3b804730b8b682135179dafb4975ed91b71a73a8 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Wed, 5 Aug 2026 10:35:52 +0300 Subject: [PATCH 10/24] multikernel: serialize CPU ownership transfers Protect CPU membership and pool ownership with a dedicated mutex nested inside an operation-wide transaction lock. Keep add and remove transactions serialized across reservation, acknowledgment, and repark so a CPU cannot be transferred twice. Signed-off-by: Nikolay Nikolaev --- include/linux/multikernel.h | 5 +- kernel/multikernel/core.c | 105 ++++++++++++---- kernel/multikernel/hotplug.c | 208 ++++++++++++++++++++++--------- kernel/multikernel/instance_dt.c | 19 ++- kernel/multikernel/internal.h | 5 + 5 files changed, 254 insertions(+), 88 deletions(-) diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index bc83af950640c3..3c648aabfef564 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include @@ -61,13 +60,15 @@ bool mk_cpu_set_del(struct mk_cpu_set *set, mk_phys_cpu_t id); bool mk_cpu_set_contains(const struct mk_cpu_set *set, mk_phys_cpu_t id); int mk_cpu_set_copy(struct mk_cpu_set *dst, const struct mk_cpu_set *src); int mk_cpu_set_format(char *buf, size_t size, const struct mk_cpu_set *set); - unsigned int mk_cpu_set_count(const struct mk_cpu_set *set); bool mk_cpu_set_empty(const struct mk_cpu_set *set); mk_phys_cpu_t mk_cpu_set_first(const struct mk_cpu_set *set); bool mk_cpu_set_get(const struct mk_cpu_set *set, unsigned int index, mk_phys_cpu_t *id); +void mk_cpu_transaction_lock(void); +void mk_cpu_transaction_unlock(void); + #define mk_cpu_set_for_each(i, id, set) \ for ((i) = 0; \ mk_cpu_set_get((set), (i), &(id)); \ diff --git a/kernel/multikernel/core.c b/kernel/multikernel/core.c index b8101e0f099594..427b7d96f6aa1c 100644 --- a/kernel/multikernel/core.c +++ b/kernel/multikernel/core.c @@ -16,6 +16,35 @@ #include #include "internal.h" +/* CPU moves hold the transaction lock before the ownership lock. */ +static DEFINE_MUTEX(mk_cpu_transaction_mutex); +static DEFINE_MUTEX(mk_cpu_ownership_mutex); + +void mk_cpu_transaction_lock(void) +{ + mutex_lock(&mk_cpu_transaction_mutex); +} + +void mk_cpu_transaction_unlock(void) +{ + mutex_unlock(&mk_cpu_transaction_mutex); +} + +void mk_cpu_ownership_lock(void) +{ + mutex_lock(&mk_cpu_ownership_mutex); +} + +void mk_cpu_ownership_unlock(void) +{ + mutex_unlock(&mk_cpu_ownership_mutex); +} + +void mk_cpu_ownership_assert_held(void) +{ + lockdep_assert_held(&mk_cpu_ownership_mutex); +} + static void mk_instance_return_all_cpus(struct mk_instance *instance) { if (!instance || mk_cpu_set_empty(instance->cpus)) @@ -484,6 +513,7 @@ int mk_instance_confirm_parked(struct mk_instance *instance) int mk_instance_transfer_cpus(struct mk_instance *instance, const struct mk_cpu_set *cpus) { + struct mk_cpu_set *requested; unsigned int i, requested_count; mk_phys_cpu_t phys_cpu; int unavailable = 0; @@ -495,14 +525,25 @@ int mk_instance_transfer_cpus(struct mk_instance *instance, return -EINVAL; } - requested_count = mk_cpu_set_count(cpus); + requested = mk_cpu_set_alloc(); + if (!requested) + return -ENOMEM; + + mk_cpu_transaction_lock(); + mk_cpu_ownership_lock(); + ret = mk_cpu_set_copy(requested, cpus); + if (ret) + goto out_unlock; + + requested_count = mk_cpu_set_count(requested); if (requested_count == 0) { pr_info("No CPUs requested for instance %d (%s)\n", instance->id, instance->name); - return 0; + ret = 0; + goto out_unlock; } - mk_cpu_set_for_each(i, phys_cpu, cpus) { + mk_cpu_set_for_each(i, phys_cpu, requested) { if (!mk_cpu_set_contains(mk_pool->cpus, phys_cpu)) { pr_err("CPU %llu not available in the pool\n", phys_cpu); @@ -519,15 +560,16 @@ int mk_instance_transfer_cpus(struct mk_instance *instance, if (unavailable > 0) { pr_err("Instance %d (%s): %d CPUs are not available\n", - instance->id, instance->name, unavailable); - return -EBUSY; + instance->id, instance->name, unavailable); + ret = -EBUSY; + goto out_unlock; } ret = mk_cpu_set_reserve(instance->cpus, requested_count); if (ret) - return ret; + goto out_unlock; - mk_cpu_set_for_each(i, phys_cpu, cpus) { + mk_cpu_set_for_each(i, phys_cpu, requested) { mk_cpu_set_del(mk_pool->cpus, phys_cpu); mk_cpu_set_add(instance->cpus, phys_cpu); } @@ -536,7 +578,12 @@ int mk_instance_transfer_cpus(struct mk_instance *instance, pr_info("Transferred %u CPUs from pool to instance %d (%s): %s\n", requested_count, instance->id, instance->name, buf); - return 0; + ret = 0; +out_unlock: + mk_cpu_ownership_unlock(); + mk_cpu_transaction_unlock(); + mk_cpu_set_free(requested); + return ret; } /** @@ -552,6 +599,7 @@ int mk_instance_transfer_cpus(struct mk_instance *instance, int mk_instance_return_cpus(struct mk_instance *instance, const struct mk_cpu_set *cpus) { + struct mk_cpu_set *requested; unsigned int i, requested_count; mk_phys_cpu_t phys_cpu; int not_found = 0; @@ -563,15 +611,26 @@ int mk_instance_return_cpus(struct mk_instance *instance, return -EINVAL; } - requested_count = mk_cpu_set_count(cpus); + requested = mk_cpu_set_alloc(); + if (!requested) + return -ENOMEM; + + mk_cpu_transaction_lock(); + mk_cpu_ownership_lock(); + ret = mk_cpu_set_copy(requested, cpus); + if (ret) + goto out_unlock; + + requested_count = mk_cpu_set_count(requested); if (requested_count == 0) { pr_info("No CPUs requested to return from instance %d (%s)\n", instance->id, instance->name); - return 0; + ret = 0; + goto out_unlock; } /* Validate all CPUs are assigned to this instance */ - mk_cpu_set_for_each(i, phys_cpu, cpus) { + mk_cpu_set_for_each(i, phys_cpu, requested) { if (!mk_cpu_set_contains(instance->cpus, phys_cpu)) { pr_err("CPU %llu not assigned to instance %d (%s)\n", phys_cpu, instance->id, instance->name); @@ -581,23 +640,18 @@ int mk_instance_return_cpus(struct mk_instance *instance, if (not_found > 0) { pr_err("Instance %d (%s): %d CPUs are not assigned to this instance\n", - instance->id, instance->name, not_found); - return -EINVAL; + instance->id, instance->name, not_found); + ret = -EINVAL; + goto out_unlock; } ret = mk_cpu_set_reserve(mk_pool->cpus, requested_count); if (ret) - return ret; + goto out_unlock; - mk_cpu_set_format(buf, sizeof(buf), cpus); + mk_cpu_set_format(buf, sizeof(buf), requested); - /* - * @cpus may alias instance->cpus (returning everything on - * teardown), so walk it back-to-front: a deletion then never - * shifts entries the walk has yet to visit. - */ - for (i = requested_count; i-- > 0; ) { - phys_cpu = cpus->ids[i]; + mk_cpu_set_for_each(i, phys_cpu, requested) { mk_cpu_set_add(mk_pool->cpus, phys_cpu); mk_cpu_set_del(instance->cpus, phys_cpu); } @@ -605,7 +659,12 @@ int mk_instance_return_cpus(struct mk_instance *instance, pr_info("Returned %u CPUs from instance %d (%s) to the pool: %s\n", requested_count, instance->id, instance->name, buf); - return 0; + ret = 0; +out_unlock: + mk_cpu_ownership_unlock(); + mk_cpu_transaction_unlock(); + mk_cpu_set_free(requested); + return ret; } /** diff --git a/kernel/multikernel/hotplug.c b/kernel/multikernel/hotplug.c index 87a308faa7e0ec..d91ea445fe5298 100644 --- a/kernel/multikernel/hotplug.c +++ b/kernel/multikernel/hotplug.c @@ -77,25 +77,6 @@ struct mk_cpu_hotplug_work { u32 operation; /* MK_RES_CPU_ADD or MK_RES_CPU_REMOVE */ }; -/* - * Ownership tracking for this kernel's own hotplug: mk_self->cpus - * is the set of CPUs this kernel owns, in the host and in spawn kernels - * alike. The assignable-pool bookkeeping (mk_pool) is not done here; - * it belongs to the mk_send_cpu_* initiator paths of the kernel that - * manages the pool. - */ -static void mk_account_cpu_online(mk_phys_cpu_t cpu_id) -{ - if (mk_self->cpus && mk_cpu_set_add(mk_self->cpus, cpu_id)) - pr_warn("Multikernel hotplug: Failed to track CPU %llu\n", - cpu_id); -} - -static void mk_account_cpu_offline(mk_phys_cpu_t cpu_id) -{ - mk_cpu_set_del(mk_self->cpus, cpu_id); -} - /** * Search present CPUs (not possible CPUs) to find the logical CPU with matching * physical ID. Using present CPUs is important because topology can change during @@ -124,6 +105,7 @@ static int mk_do_cpu_add(mk_phys_cpu_t cpu_id, u32 numa_node, u32 flags) pr_info("Multikernel hotplug: Adding CPU %llu (numa=%d, flags=0x%x)\n", cpu_id, (int)numa_node, flags); + mk_cpu_transaction_lock(); logical_cpu = mk_cpu_to_logical(cpu_id); if (logical_cpu < 0) { /* @@ -136,7 +118,8 @@ static int mk_do_cpu_add(mk_phys_cpu_t cpu_id, u32 numa_node, u32 flags) if (logical_cpu < 0) { pr_err("Multikernel hotplug: CPU %llu is not in this kernel's pool\n", cpu_id); - return -ENODEV; + ret = -ENODEV; + goto unlock_transaction; } set_cpu_present(logical_cpu, true); } @@ -144,18 +127,37 @@ static int mk_do_cpu_add(mk_phys_cpu_t cpu_id, u32 numa_node, u32 flags) if (cpu_online(logical_cpu)) { pr_warn("Multikernel hotplug: CPU %d (phys %llu) already online\n", logical_cpu, cpu_id); - mk_account_cpu_online(cpu_id); - return 0; + mk_cpu_ownership_lock(); + if (mk_self->cpus) { + ret = mk_cpu_set_add(mk_self->cpus, cpu_id); + if (ret) + pr_warn("Multikernel hotplug: Failed to track CPU %llu in root pool\n", + cpu_id); + } + mk_cpu_ownership_unlock(); + ret = 0; + goto unlock_transaction; + } + + mk_cpu_ownership_lock(); + if (mk_self->cpus) { + ret = mk_cpu_set_reserve(mk_self->cpus, 1); + if (ret) { + mk_cpu_ownership_unlock(); + goto unlock_transaction; + } } + mk_cpu_ownership_unlock(); if (!get_cpu_device(logical_cpu)) { struct cpu *c = &per_cpu(cpu_devices, logical_cpu); + c->hotpluggable = true; ret = register_cpu(c, logical_cpu); if (ret) { pr_err("Multikernel hotplug: Failed to register CPU %d: %d\n", logical_cpu, ret); - return ret; + goto unlock_transaction; } } @@ -163,10 +165,19 @@ static int mk_do_cpu_add(mk_phys_cpu_t cpu_id, u32 numa_node, u32 flags) if (ret < 0) { pr_err("Multikernel hotplug: Failed to add CPU %d (phys %llu): %d\n", logical_cpu, cpu_id, ret); - return ret; + goto unlock_transaction; } - mk_account_cpu_online(cpu_id); + mk_cpu_ownership_lock(); + if (mk_self->cpus) { + ret = mk_cpu_set_add(mk_self->cpus, cpu_id); + if (ret) + pr_warn("Multikernel hotplug: Failed to track CPU %llu in root pool\n", + cpu_id); + } + mk_cpu_ownership_unlock(); + if (ret) + goto unlock_transaction; /* Track the operation for potential rollback */ op = kzalloc(sizeof(*op), GFP_KERNEL); @@ -183,7 +194,10 @@ static int mk_do_cpu_add(mk_phys_cpu_t cpu_id, u32 numa_node, u32 flags) pr_info("Multikernel hotplug: Successfully added CPU %d (phys %llu)\n", logical_cpu, cpu_id); - return 0; + ret = 0; +unlock_transaction: + mk_cpu_transaction_unlock(); + return ret; } static int mk_do_cpu_remove(mk_phys_cpu_t cpu_id) @@ -191,34 +205,56 @@ static int mk_do_cpu_remove(mk_phys_cpu_t cpu_id) int logical_cpu; int ret; struct mk_hotplug_op *op; + bool tracked; + mk_cpu_transaction_lock(); logical_cpu = mk_cpu_to_logical(cpu_id); if (logical_cpu < 0) { pr_err("Multikernel hotplug: Physical CPU %llu not found\n", cpu_id); - return -ENODEV; + ret = -ENODEV; + goto unlock_transaction; } + mk_cpu_ownership_lock(); + tracked = mk_self->cpus && + mk_cpu_set_contains(mk_self->cpus, cpu_id); + if (!cpu_online(logical_cpu)) { pr_warn("Multikernel hotplug: CPU %d (phys %llu) already offline\n", logical_cpu, cpu_id); - mk_account_cpu_offline(cpu_id); - return 0; + mk_cpu_set_del(mk_self->cpus, cpu_id); + mk_cpu_ownership_unlock(); + ret = 0; + goto unlock_transaction; + } + + if (!tracked) { + pr_err("Multikernel hotplug: CPU %llu is not tracked in root pool\n", + cpu_id); + mk_cpu_ownership_unlock(); + ret = -EINVAL; + goto unlock_transaction; } /* Don't allow removing CPU 0 (boot processor) */ if (logical_cpu == 0) { pr_err("Multikernel hotplug: Cannot remove boot CPU\n"); - return -EINVAL; + mk_cpu_ownership_unlock(); + ret = -EINVAL; + goto unlock_transaction; } + mk_cpu_ownership_unlock(); ret = depart_cpu(logical_cpu); if (ret < 0) { pr_err("Multikernel hotplug: Failed to remove CPU %d (phys %llu): %d\n", logical_cpu, cpu_id, ret); - return ret; + goto unlock_transaction; } - mk_account_cpu_offline(cpu_id); + mk_cpu_ownership_lock(); + mk_cpu_set_del(mk_self->cpus, cpu_id); + mk_cpu_ownership_unlock(); /* * Clear CPU from present mask to prevent host kernel from trying @@ -241,7 +277,10 @@ static int mk_do_cpu_remove(mk_phys_cpu_t cpu_id) pr_info("Multikernel hotplug: Successfully removed CPU %d (phys %llu)\n", logical_cpu, cpu_id); - return 0; + ret = 0; +unlock_transaction: + mk_cpu_transaction_unlock(); + return ret; } static void mk_cpu_add_work_fn(struct work_struct *work) @@ -1237,6 +1276,7 @@ int mk_send_cpu_remove(int instance_id, mk_phys_cpu_t cpu_id) if (target_instance->state != MK_STATE_ACTIVE) { struct mk_cpu_set cpus = { .nr = 1, .cap = 1, .ids = &cpu_id }; + raw_spin_lock_init(&cpus.lock); /* * A CPU the instance has already run on is parked on that * instance's context. Bring it back to the host slot before @@ -1262,26 +1302,39 @@ int mk_send_cpu_remove(int instance_id, mk_phys_cpu_t cpu_id) goto out; } - ret = mk_cpu_set_reserve(mk_pool->cpus, 1); - if (ret) - goto out; - + mk_cpu_transaction_lock(); pending = mk_msg_pending_add(MK_MSG_RESOURCE, MK_RES_CPU_REMOVE, cpu_id); if (!pending) { ret = -ENOMEM; - goto out; + goto unlock_transaction; } - ret = mk_send_message(instance_id, MK_MSG_RESOURCE, MK_RES_CPU_REMOVE, - &payload, sizeof(payload)); + mk_cpu_ownership_lock(); + if (!mk_cpu_set_contains(target_instance->cpus, cpu_id)) { + pr_err("Multikernel hotplug: CPU %llu not assigned to instance %d\n", + cpu_id, instance_id); + mk_msg_pending_wait(pending, 0); + ret = -EINVAL; + goto unlock_ownership; + } + + ret = mk_cpu_set_reserve(mk_pool->cpus, 1); + if (ret) { + mk_msg_pending_wait(pending, 0); + goto unlock_ownership; + } + mk_cpu_ownership_unlock(); + + ret = mk_send_message(target_instance->id, MK_MSG_RESOURCE, + MK_RES_CPU_REMOVE, &payload, sizeof(payload)); if (ret < 0) { mk_msg_pending_wait(pending, 0); /* Immediate cleanup */ - goto out; + goto unlock_transaction; } ret = mk_msg_pending_wait(pending, 10000); if (ret < 0) - goto out; + goto unlock_transaction; /* * The spawn kernel parked the CPU on its own context when it went @@ -1295,13 +1348,27 @@ int mk_send_cpu_remove(int instance_id, mk_phys_cpu_t cpu_id) if (ret < 0) { pr_err("Multikernel hotplug: CPU %llu offline in instance %d but not reparked to host: %d\n", cpu_id, instance_id, ret); - goto out; + goto unlock_transaction; } + mk_cpu_ownership_lock(); + if (!mk_cpu_set_contains(target_instance->cpus, cpu_id)) { + ret = -ESTALE; + goto unlock_ownership; + } + ret = mk_cpu_set_add(mk_pool->cpus, cpu_id); + if (ret) { + pr_warn("Multikernel hotplug: Failed to track CPU %llu in pool\n", + cpu_id); + goto unlock_ownership; + } mk_cpu_set_del(target_instance->cpus, cpu_id); - mk_cpu_set_add(mk_pool->cpus, cpu_id); ret = 0; +unlock_ownership: + mk_cpu_ownership_unlock(); +unlock_transaction: + mk_cpu_transaction_unlock(); out: mk_instance_put(target_instance); return ret; @@ -1346,26 +1413,41 @@ int mk_send_cpu_add(int instance_id, mk_phys_cpu_t cpu_id, u32 numa_node, u32 fl instance_id); return -ENODEV; } + if (arch_cpu_from_physical_id(cpu_id) == 0) { + pr_err("Multikernel hotplug: CPU %llu is reserved for host control\n", + cpu_id); + ret = -EINVAL; + goto out; + } /* For non-running instances, transfer CPU from root using existing API */ if (target_instance->state != MK_STATE_ACTIVE) { struct mk_cpu_set cpus = { .nr = 1, .cap = 1, .ids = &cpu_id }; + raw_spin_lock_init(&cpus.lock); ret = mk_instance_transfer_cpus(target_instance, &cpus); goto out; } - /* - * Only a CPU from the assignable pool is parked on the host slot; - * publishing a wakeup for any other CPU can only time out. - */ + if (!mk_pool) { + ret = -ENODEV; + goto out; + } + + mk_cpu_transaction_lock(); + mk_cpu_ownership_lock(); if (!mk_cpu_set_contains(mk_pool->cpus, cpu_id)) { - pr_err("Multikernel hotplug: CPU %llu is not in this kernel's pool\n", + pr_err("Multikernel hotplug: CPU %llu not available in the pool\n", cpu_id); ret = -EBUSY; - goto out; + goto unlock_ownership; } + ret = mk_cpu_set_reserve(target_instance->cpus, 1); + if (ret) + goto unlock_ownership; + mk_cpu_ownership_unlock(); + /* * The CPU is parked on the host slot, where the spawn kernel's * secondary wakeup cannot reach it. Point it at the instance's @@ -1375,22 +1457,22 @@ int mk_send_cpu_add(int instance_id, mk_phys_cpu_t cpu_id, u32 numa_node, u32 fl if (ret < 0) { pr_err("Multikernel hotplug: Failed to repark CPU %llu to instance %d: %d\n", cpu_id, instance_id, ret); - goto out; + goto unlock_transaction; } pending = mk_msg_pending_add(MK_MSG_RESOURCE, MK_RES_CPU_ADD, cpu_id); if (!pending) { mk_repark_cpu_to_host(target_instance, cpu_id); ret = -ENOMEM; - goto out; + goto unlock_transaction; } - ret = mk_send_message(instance_id, MK_MSG_RESOURCE, MK_RES_CPU_ADD, - &payload, sizeof(payload)); + ret = mk_send_message(target_instance->id, MK_MSG_RESOURCE, + MK_RES_CPU_ADD, &payload, sizeof(payload)); if (ret < 0) { mk_msg_pending_wait(pending, 0); /* Immediate cleanup */ mk_repark_cpu_to_host(target_instance, cpu_id); - goto out; + goto unlock_transaction; } ret = mk_msg_pending_wait(pending, 10000); @@ -1402,15 +1484,27 @@ int mk_send_cpu_add(int instance_id, mk_phys_cpu_t cpu_id, u32 numa_node, u32 fl * watching the context and this times out harmlessly. */ mk_repark_cpu_to_host(target_instance, cpu_id); - goto out; + goto unlock_transaction; } - if (mk_cpu_set_add(target_instance->cpus, cpu_id)) + mk_cpu_ownership_lock(); + if (!mk_cpu_set_contains(mk_pool->cpus, cpu_id)) { + ret = -ESTALE; + goto unlock_ownership; + } + ret = mk_cpu_set_add(target_instance->cpus, cpu_id); + if (ret) { pr_warn("Multikernel hotplug: Failed to track CPU %llu in instance %d\n", cpu_id, instance_id); + goto unlock_ownership; + } mk_cpu_set_del(mk_pool->cpus, cpu_id); ret = 0; +unlock_ownership: + mk_cpu_ownership_unlock(); +unlock_transaction: + mk_cpu_transaction_unlock(); out: mk_instance_put(target_instance); return ret; diff --git a/kernel/multikernel/instance_dt.c b/kernel/multikernel/instance_dt.c index 835922bc491075..85627764afab4c 100644 --- a/kernel/multikernel/instance_dt.c +++ b/kernel/multikernel/instance_dt.c @@ -463,12 +463,19 @@ int __init mk_instance_restore_from_manifest(void) pr_err("Failed to allocate the self instance\n"); return -ENOMEM; } - /* Initially, root owns all online CPUs (physical IDs) */ - for_each_online_cpu(cpu) { - if (mk_cpu_set_add(instance->cpus, - arch_cpu_physical_id(cpu))) - pr_warn("Failed to add CPU %d to the self instance\n", - cpu); + /* + * Root owns every enumerated CPU, including APs that become online + * only after early initcalls complete. + */ + for_each_present_cpu(cpu) { + ret = mk_cpu_set_add(instance->cpus, + arch_cpu_physical_id(cpu)); + if (ret) { + pr_err("Failed to track CPU %d in the self instance: %d\n", + cpu, ret); + mk_instance_free(instance); + return ret; + } } mk_cpu_set_format(cpus_buf, sizeof(cpus_buf), instance->cpus); pr_info("Self instance initialized with CPUs (physical): %s\n", diff --git a/kernel/multikernel/internal.h b/kernel/multikernel/internal.h index 137001c8172524..26c3532e3a244d 100644 --- a/kernel/multikernel/internal.h +++ b/kernel/multikernel/internal.h @@ -47,6 +47,11 @@ int mk_dt_generate_instance_dtb(struct mk_instance *instance, int mk_pci_parse_bdf(const char *pci_id, int len, u16 *domain, u8 *bus, u8 *slot, u8 *func); +/* CPU ownership serialization: transaction must be acquired first. */ +void mk_cpu_ownership_lock(void); +void mk_cpu_ownership_unlock(void); +void mk_cpu_ownership_assert_held(void); + /* pci.c */ int mk_pci_lease_system_init(void); void mk_pci_lease_system_cleanup(void); From 27750f25d440f053639e8b19a70b33b5e3d7b097 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Wed, 12 Aug 2026 13:51:10 +0300 Subject: [PATCH 11/24] multikernel: make IPI publication ordered and recoverable Serialize shared-ring producers with a bounded owner-aware gate. Preserve FIFO publication and recover a gate only after its producer CPU is known to be parked. The gate and slot-state protocol change the private shared transport layout. Add an exact pre-launch layout and protocol check at this boundary, then require a transport initialization acknowledgment after both rings have been validated and before marking the instance active. Fail invalid manifests and missing acknowledgments closed. A spawn started by a host without the pre-launch check validates the boot-context anchor before using any shifted field and enters a local interrupt-disabled halt loop on mismatch without trusting shared park state or touching reset and APIC hardware. Signed-off-by: Nikolay Nikolaev --- arch/x86/kernel/kexec-vmlinux.c | 4 +- include/linux/multikernel.h | 89 ++++++++ include/linux/multikernel_abi.h | 4 +- kernel/multikernel/ipi.c | 358 ++++++++++++++++++++++++++++++++ 4 files changed, 452 insertions(+), 3 deletions(-) diff --git a/arch/x86/kernel/kexec-vmlinux.c b/arch/x86/kernel/kexec-vmlinux.c index ef5fc7ce6941b1..727144d65f0a16 100644 --- a/arch/x86/kernel/kexec-vmlinux.c +++ b/arch/x86/kernel/kexec-vmlinux.c @@ -65,6 +65,7 @@ struct elf_kernel_info { * The note type carries the generation; the descriptor remains one u64. */ #define MK_VMLINUX_LEGACY_NOTE_TYPE 0x4d4b +#define MK_VMLINUX_PREVIOUS_NOTE_TYPE 0x4d4b0002 static int find_multikernel_entry_note(const void *buf, size_t len, const Elf64_Ehdr *ehdr, @@ -108,7 +109,8 @@ static int find_multikernel_entry_note(const void *buf, size_t len, *note_entry); return 0; } - if (nhdr->n_type == MK_VMLINUX_LEGACY_NOTE_TYPE) + if (nhdr->n_type == MK_VMLINUX_LEGACY_NOTE_TYPE || + nhdr->n_type == MK_VMLINUX_PREVIOUS_NOTE_TYPE) legacy = true; } ptr += note_size; diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index 3c648aabfef564..9b65b2c613843c 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -84,10 +84,54 @@ void mk_cpu_transaction_unlock(void); /* IPI ring buffer size - must be power of 2 for efficient modulo */ #define MK_IPI_RING_SIZE 64 +#define MK_REPLY_SLOTS 16 +#define MK_REPLY_STATE_BITS 3 + +enum mk_reply_state { + MK_REPLY_FREE = 0, + MK_REPLY_RESERVED, + MK_REPLY_WRITING, + MK_REPLY_EXECUTING, + MK_REPLY_COMMITTED, + MK_REPLY_READY, + MK_REPLY_ABANDONED, +}; + +enum mk_reply_kind { + MK_REPLY_PCI_CFG = 1, + MK_REPLY_PCI_IRQ, +}; + +struct mk_reply_slot { + atomic64_t state_generation; + u64 request_id; + u32 kind; + s32 status; + u32 value; + u32 reserved; +}; + +struct mk_reply_table { + struct mk_reply_slot slots[MK_REPLY_SLOTS]; + atomic_t late_replies; + atomic_t cancelled_slots; + atomic_t atomic_timeouts; + atomic_t indeterminate_timeouts; + atomic_t occupied_failures; +}; + +struct mk_reply_handle { + u32 slot; + u32 kind; + u64 request_id; + u64 generation; +}; + /* Data structure for passing parameters via IPI */ struct mk_ipi_data { u32 ready; u64 sender_cpu; /* Physical ID of the CPU that sent this IPI */ + s32 sender_instance_id; /* Receiver-authenticated duplex peer */ unsigned int type; /* User-defined type identifier */ size_t data_size; /* Size of the data */ char buffer[MK_MAX_DATA_SIZE]; /* Actual data buffer */ @@ -134,8 +178,30 @@ struct mk_shared_data { u32 reserved; u64 parent_doorbell_cpu; u64 child_doorbell_cpu; + /* Changes on every launch; zero means the link has not been launched. */ + u64 spawn_epoch; + /* Generation-tagged synchronous replies, independent of ring progress. */ + struct mk_reply_table replies; }; +static inline void mk_reply_table_reset(struct mk_reply_table *table) +{ + unsigned int i; + + for (i = 0; i < MK_REPLY_SLOTS; i++) { + atomic64_set(&table->slots[i].state_generation, MK_REPLY_FREE); + WRITE_ONCE(table->slots[i].request_id, 0); + WRITE_ONCE(table->slots[i].kind, 0); + WRITE_ONCE(table->slots[i].status, 0); + WRITE_ONCE(table->slots[i].value, 0); + } + atomic_set(&table->late_replies, 0); + atomic_set(&table->cancelled_slots, 0); + atomic_set(&table->atomic_timeouts, 0); + atomic_set(&table->indeterminate_timeouts, 0); + atomic_set(&table->occupied_failures, 0); +} + static inline void mk_ipi_ring_reset(struct mk_ipi_ring *ring) { unsigned int i; @@ -146,12 +212,18 @@ static inline void mk_ipi_ring_reset(struct mk_ipi_ring *ring) static inline void mk_shared_data_reset(struct mk_shared_data *shared) { + unsigned int i; + mk_ipi_ring_reset(&shared->to_child); mk_ipi_ring_reset(&shared->to_parent); WRITE_ONCE(shared->force_halt, 0); + for (i = 0; i < MK_PARKED_MAX; i++) + WRITE_ONCE(shared->parked[i], 0); + mk_reply_table_reset(&shared->replies); } struct mk_ipi_endpoint { + struct mk_instance *peer; struct mk_ipi_ring *tx; struct mk_ipi_ring *rx; raw_spinlock_t tx_lock; @@ -216,6 +288,23 @@ void mk_ipi_link_reset(struct mk_instance *instance, int parent_id, int child_id, mk_phys_cpu_t parent_cpu, mk_phys_cpu_t child_cpu); void mk_ipi_handlers_enable(void); +int mk_reply_reserve(struct mk_shared_data *shared, u32 kind, u64 request_id, + struct mk_reply_handle *reply); +int mk_reply_claim(struct mk_instance *instance, + const struct mk_reply_handle *reply); +int mk_reply_begin_execute(struct mk_instance *instance, + const struct mk_reply_handle *reply); +int mk_reply_publish(struct mk_instance *instance, + const struct mk_reply_handle *reply, s32 status, u32 value); +int mk_reply_wait_atomic(struct mk_shared_data *shared, + struct mk_reply_handle *reply, unsigned int timeout_us, + s32 *status, u32 *value); +int mk_reply_wait(struct mk_shared_data *shared, + struct mk_reply_handle *reply, unsigned int timeout_ms, + s32 *status, u32 *value); +void mk_reply_release(struct mk_shared_data *shared, + struct mk_reply_handle *reply); +void mk_reply_scan(struct mk_shared_data *shared); /* * Multikernel Messaging System diff --git a/include/linux/multikernel_abi.h b/include/linux/multikernel_abi.h index 3d9bb5fdeee61e..9b5f5978ebe53d 100644 --- a/include/linux/multikernel_abi.h +++ b/include/linux/multikernel_abi.h @@ -3,7 +3,7 @@ #define _LINUX_MULTIKERNEL_ABI_H /* One generation marker shared by the spawn context and image capabilities. */ -#define MK_BOOT_CONTEXT_MAGIC 0x4d4b0002 -#define MK_VMLINUX_NOTE_TYPE 0x4d4b0002 +#define MK_BOOT_CONTEXT_MAGIC 0x4d4b0003 +#define MK_VMLINUX_NOTE_TYPE 0x4d4b0003 #endif /* _LINUX_MULTIKERNEL_ABI_H */ diff --git a/kernel/multikernel/ipi.c b/kernel/multikernel/ipi.c index b05fffe6019738..c069b1a4918edd 100644 --- a/kernel/multikernel/ipi.c +++ b/kernel/multikernel/ipi.c @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include "internal.h" static struct mk_ipi_handler *mk_handlers; @@ -16,6 +18,25 @@ static DEFINE_RAW_SPINLOCK(mk_ipi_endpoints_lock); static bool mk_handlers_ready; static DEFINE_RATELIMIT_STATE(mk_ipi_full_rs, DEFAULT_RATELIMIT_INTERVAL, DEFAULT_RATELIMIT_BURST); +static DECLARE_WAIT_QUEUE_HEAD(mk_reply_waitq); + +#define MK_REPLY_STATE_MASK (BIT(MK_REPLY_STATE_BITS) - 1) +#define MK_REPLY_GENERATION_MAX (U64_MAX >> MK_REPLY_STATE_BITS) + +static u64 mk_reply_token(u64 generation, enum mk_reply_state state) +{ + return generation << MK_REPLY_STATE_BITS | state; +} + +static u64 mk_reply_generation(u64 token) +{ + return token >> MK_REPLY_STATE_BITS; +} + +static enum mk_reply_state mk_reply_state(u64 token) +{ + return token & MK_REPLY_STATE_MASK; +} static struct mk_shared_data *mk_instance_ipi_area(struct mk_instance *instance) { @@ -38,6 +59,7 @@ int mk_ipi_endpoint_init(struct mk_instance *instance, bool parent_side) return -ENODEV; if (endpoint->registered) return 0; + endpoint->peer = instance; endpoint->tx = parent_side ? &instance->ipi_data->to_child : &instance->ipi_data->to_parent; endpoint->rx = parent_side ? &instance->ipi_data->to_parent : @@ -88,11 +110,16 @@ void mk_ipi_link_reset(struct mk_instance *instance, int parent_id, mk_phys_cpu_t child_cpu) { struct mk_shared_data *shared = mk_instance_ipi_area(instance); + u64 epoch; if (!shared) return; + epoch = READ_ONCE(shared->spawn_epoch) + 1; + if (!epoch) + epoch = 1; mk_ipi_endpoint_unregister(instance); mk_shared_data_reset(shared); + WRITE_ONCE(shared->spawn_epoch, epoch); WRITE_ONCE(shared->parent_id, parent_id); WRITE_ONCE(shared->child_id, child_id); WRITE_ONCE(shared->parent_doorbell_cpu, parent_cpu); @@ -100,6 +127,334 @@ void mk_ipi_link_reset(struct mk_instance *instance, int parent_id, mk_ipi_endpoint_init(instance, true); } +int mk_reply_reserve(struct mk_shared_data *shared, u32 kind, u64 request_id, + struct mk_reply_handle *reply) +{ + struct mk_reply_table *table; + unsigned int i; + int ret; + + if (!reply || !request_id || !kind) + return -EINVAL; + ret = shared ? 0 : -ENODEV; + if (ret) + return ret; + + table = &shared->replies; + for (i = 0; i < MK_REPLY_SLOTS; i++) { + struct mk_reply_slot *slot = &table->slots[i]; + u64 generation; + u64 claim; + u64 old; + + old = atomic64_read(&slot->state_generation); + if (mk_reply_state(old) != MK_REPLY_FREE) + continue; + generation = mk_reply_generation(old) + 1; + if (!generation || generation > MK_REPLY_GENERATION_MAX) + generation = 1; + claim = mk_reply_token(generation, MK_REPLY_WRITING); + if (atomic64_cmpxchg_acquire(&slot->state_generation, old, + claim) != old) + continue; + + WRITE_ONCE(slot->request_id, request_id); + WRITE_ONCE(slot->kind, kind); + WRITE_ONCE(slot->status, -ETIMEDOUT); + WRITE_ONCE(slot->value, ~0U); + atomic64_set_release(&slot->state_generation, + mk_reply_token(generation, + MK_REPLY_RESERVED)); + reply->slot = i; + reply->kind = kind; + reply->request_id = request_id; + reply->generation = generation; + return 0; + } + + atomic_inc(&table->occupied_failures); + return -ENOSPC; +} + +static int mk_reply_take_ready(struct mk_shared_data *shared, + struct mk_reply_handle *reply, + s32 *status, u32 *value) +{ + struct mk_reply_slot *slot = &shared->replies.slots[reply->slot]; + u64 ready = mk_reply_token(reply->generation, MK_REPLY_READY); + u64 free = mk_reply_token(reply->generation, MK_REPLY_FREE); + + if (atomic64_read_acquire(&slot->state_generation) != ready) + return -EAGAIN; + if (READ_ONCE(slot->request_id) != reply->request_id || + READ_ONCE(slot->kind) != reply->kind) + return -EPROTO; + if (status) + *status = READ_ONCE(slot->status); + if (value) + *value = READ_ONCE(slot->value); + if (atomic64_cmpxchg_release(&slot->state_generation, ready, free) != + ready) + return -EAGAIN; + return 0; +} + +static bool mk_reply_cancel(struct mk_shared_data *shared, + struct mk_reply_handle *reply, bool atomic_timeout) +{ + struct mk_reply_table *table = &shared->replies; + struct mk_reply_slot *slot = &table->slots[reply->slot]; + u64 reserved = mk_reply_token(reply->generation, MK_REPLY_RESERVED); + u64 writing = mk_reply_token(reply->generation, MK_REPLY_WRITING); + u64 executing = mk_reply_token(reply->generation, MK_REPLY_EXECUTING); + u64 committed = mk_reply_token(reply->generation, MK_REPLY_COMMITTED); + u64 abandoned = mk_reply_token(reply->generation, MK_REPLY_ABANDONED); + u64 ready = mk_reply_token(reply->generation, MK_REPLY_READY); + u64 free = mk_reply_token(reply->generation, MK_REPLY_FREE); + u64 token; + + if (atomic64_cmpxchg_release(&slot->state_generation, reserved, free) == + reserved) + goto cancelled; + + for (;;) { + token = atomic64_read_acquire(&slot->state_generation); + if (token == ready) + return true; + if (token == writing && + atomic64_cmpxchg_release(&slot->state_generation, writing, + abandoned) == writing) + break; + if (token == executing && + atomic64_cmpxchg_release(&slot->state_generation, executing, + committed) == executing) { + atomic_inc(&table->indeterminate_timeouts); + goto timed_out; + } + if (token != writing && token != executing) + break; + } + +cancelled: + atomic_inc(&table->cancelled_slots); +timed_out: + if (atomic_timeout) + atomic_inc(&table->atomic_timeouts); + return false; +} + +static bool mk_reply_wait_done(struct mk_shared_data *shared, + const struct mk_reply_handle *reply) +{ + struct mk_reply_slot *slot = &shared->replies.slots[reply->slot]; + u64 token; + + token = atomic64_read_acquire(&slot->state_generation); + return token == mk_reply_token(reply->generation, MK_REPLY_READY) || + mk_reply_generation(token) != reply->generation || + mk_reply_state(token) == MK_REPLY_FREE; +} + +int mk_reply_wait_atomic(struct mk_shared_data *shared, + struct mk_reply_handle *reply, unsigned int timeout_us, + s32 *status, u32 *value) +{ + u64 deadline; + + if (!shared || !reply || reply->slot >= MK_REPLY_SLOTS) + return -EINVAL; + deadline = ktime_get_mono_fast_ns() + (u64)timeout_us * NSEC_PER_USEC; + for (;;) { + int ret = mk_reply_take_ready(shared, reply, status, value); + + if (!ret) + return 0; + if (ret != -EAGAIN) + return ret; + if (ktime_get_mono_fast_ns() >= deadline) + break; + cpu_relax(); + } + + if (mk_reply_cancel(shared, reply, true)) + return mk_reply_take_ready(shared, reply, status, value); + return -ETIMEDOUT; +} + +int mk_reply_wait(struct mk_shared_data *shared, + struct mk_reply_handle *reply, unsigned int timeout_ms, + s32 *status, u32 *value) +{ + long waited; + int ret; + + if (!shared || !reply || reply->slot >= MK_REPLY_SLOTS) + return -EINVAL; + waited = wait_event_timeout(mk_reply_waitq, + mk_reply_wait_done(shared, reply), + msecs_to_jiffies(timeout_ms)); + if (!waited) { + if (mk_reply_cancel(shared, reply, false)) + return mk_reply_take_ready(shared, reply, status, value); + return -ETIMEDOUT; + } + ret = mk_reply_take_ready(shared, reply, status, value); + if (ret == -EAGAIN) + ret = -ESTALE; + return ret; +} + +void mk_reply_release(struct mk_shared_data *shared, + struct mk_reply_handle *reply) +{ + if (!shared || !reply || reply->slot >= MK_REPLY_SLOTS) + return; + mk_reply_cancel(shared, reply, false); +} + +int mk_reply_claim(struct mk_instance *instance, + const struct mk_reply_handle *reply) +{ + struct mk_shared_data *shared; + struct mk_reply_slot *slot; + u64 writing; + u64 abandoned; + u64 reserved; + u64 free; + int ret; + + if (!instance || !reply || reply->slot >= MK_REPLY_SLOTS) + return -EINVAL; + shared = instance->ipi_data; + ret = shared ? 0 : -ENODEV; + if (ret) + return ret; + slot = &shared->replies.slots[reply->slot]; + reserved = mk_reply_token(reply->generation, MK_REPLY_RESERVED); + writing = mk_reply_token(reply->generation, MK_REPLY_WRITING); + abandoned = mk_reply_token(reply->generation, MK_REPLY_ABANDONED); + free = mk_reply_token(reply->generation, MK_REPLY_FREE); + if (atomic64_cmpxchg_acquire(&slot->state_generation, reserved, + writing) != reserved) { + atomic_inc(&shared->replies.late_replies); + return -ESTALE; + } + if (READ_ONCE(slot->request_id) != reply->request_id || + READ_ONCE(slot->kind) != reply->kind) { + u64 old; + + old = atomic64_cmpxchg_release(&slot->state_generation, writing, + free); + if (old == abandoned) + old = atomic64_cmpxchg_release(&slot->state_generation, + abandoned, free); + if (old == writing || old == abandoned) + wake_up_all(&mk_reply_waitq); + atomic_inc(&shared->replies.late_replies); + return -ESTALE; + } + return 0; +} + +int mk_reply_begin_execute(struct mk_instance *instance, + const struct mk_reply_handle *reply) +{ + struct mk_reply_slot *slot; + u64 writing; + u64 executing; + u64 old; + + if (!instance || !instance->ipi_data || !reply || + reply->slot >= MK_REPLY_SLOTS) + return -EINVAL; + slot = &instance->ipi_data->replies.slots[reply->slot]; + writing = mk_reply_token(reply->generation, MK_REPLY_WRITING); + executing = mk_reply_token(reply->generation, MK_REPLY_EXECUTING); + old = atomic64_cmpxchg_acquire(&slot->state_generation, writing, + executing); + if (old == writing) + return 0; + if (old == mk_reply_token(reply->generation, MK_REPLY_ABANDONED)) + return -ECANCELED; + return -ESTALE; +} + +int mk_reply_publish(struct mk_instance *instance, + const struct mk_reply_handle *reply, s32 status, u32 value) +{ + struct mk_shared_data *shared; + struct mk_reply_slot *slot; + mk_phys_cpu_t target; + u64 writing; + u64 executing; + u64 committed; + u64 abandoned; + u64 ready; + u64 free; + u64 old; + int ret; + + if (!instance || !reply || reply->slot >= MK_REPLY_SLOTS) + return -EINVAL; + shared = instance->ipi_data; + ret = shared ? 0 : -ENODEV; + if (ret) + return ret; + slot = &shared->replies.slots[reply->slot]; + writing = mk_reply_token(reply->generation, MK_REPLY_WRITING); + executing = mk_reply_token(reply->generation, MK_REPLY_EXECUTING); + committed = mk_reply_token(reply->generation, MK_REPLY_COMMITTED); + abandoned = mk_reply_token(reply->generation, MK_REPLY_ABANDONED); + ready = mk_reply_token(reply->generation, MK_REPLY_READY); + free = mk_reply_token(reply->generation, MK_REPLY_FREE); + + WRITE_ONCE(slot->status, status); + WRITE_ONCE(slot->value, value); + old = atomic64_cmpxchg_release(&slot->state_generation, executing, ready); + if (old == writing) + old = atomic64_cmpxchg_release(&slot->state_generation, writing, + ready); + if (old == abandoned) { + atomic64_set_release(&slot->state_generation, free); + atomic_inc(&shared->replies.late_replies); + return -ESTALE; + } + if (old == committed) { + atomic64_set_release(&slot->state_generation, free); + atomic_inc(&shared->replies.late_replies); + return -ESTALE; + } + if (old != executing && old != writing) { + atomic_inc(&shared->replies.late_replies); + return -EIO; + } + + target = READ_ONCE(instance->ipi_target); + if (target == MK_PHYS_CPU_INVALID) + target = mk_cpu_set_first(instance->cpus); + if (target == MK_PHYS_CPU_INVALID) + return -ENODEV; + mk_arch_send_ipi(target); + return 0; +} + +void mk_reply_scan(struct mk_shared_data *shared) +{ + unsigned int i; + + if (!shared) + return; + for (i = 0; i < MK_REPLY_SLOTS; i++) { + struct mk_reply_slot *slot = &shared->replies.slots[i]; + u64 token = atomic64_read_acquire(&slot->state_generation); + + if (mk_reply_state(token) == MK_REPLY_READY) { + wake_up_all(&mk_reply_waitq); + return; + } + } +} + struct mk_ipi_handler * multikernel_register_handler(mk_ipi_callback_t callback, void *ctx, unsigned int ipi_type) @@ -273,6 +628,7 @@ static void mk_ipi_drain_endpoint(struct mk_ipi_endpoint *endpoint) endpoint->rx_dispatching = true; raw_spin_unlock_irqrestore(&endpoint->rx_lock, flags); for (;;) { + mk_reply_scan(endpoint->peer->ipi_data); idx = endpoint->rx_tail & (MK_IPI_RING_SIZE - 1); slot = &endpoint->rx->entries[idx]; /* Pair with the producer's release publication of this slot. */ @@ -288,6 +644,8 @@ static void mk_ipi_drain_endpoint(struct mk_ipi_endpoint *endpoint) raw_spin_unlock_irqrestore(&endpoint->rx_lock, flags); return; } + /* Authenticate provenance from the registered duplex endpoint. */ + WRITE_ONCE(slot->sender_instance_id, endpoint->peer->id); mk_ipi_dispatch(slot); /* The callback must finish reading before the slot is reusable. */ smp_store_release(&slot->ready, 0); From 2c9ad68e9577cfe572b57c59e0c244202ae7b7bf Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Tue, 11 Aug 2026 03:28:14 +0300 Subject: [PATCH 12/24] multikernel: pin control routes across CPU ownership changes Cache the IRQ forwarding CPU for hardirq-safe routing. Serialize route, ownership, reload, halt, and teardown mutations while active users hold a route reference. Drain assignment IRQ producers before publishing a replacement route or reparking the old CPU. Signed-off-by: Nikolay Nikolaev --- include/linux/multikernel.h | 22 ++++++- kernel/kexec_core.c | 33 ++++++++++- kernel/multikernel/core.c | 99 ++++++++++++++++++++++++++++++-- kernel/multikernel/hotplug.c | 46 +++++++++++---- kernel/multikernel/instance_dt.c | 1 + kernel/multikernel/internal.h | 27 +++++++-- kernel/multikernel/ipi.c | 64 ++++++++++++++------- kernel/multikernel/messaging.c | 46 +++++++++++++-- net/vmw_vsock/mk_transport.c | 3 +- 9 files changed, 291 insertions(+), 50 deletions(-) diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index 9b65b2c613843c..1263d73612f8fb 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -18,6 +18,7 @@ #include #include #include +#include struct pci_bus; struct mk_instance; @@ -437,7 +438,8 @@ struct mk_shutdown_payload { * Message handler callback type */ typedef void (*mk_msg_handler_t)(u32 msg_type, u32 subtype, - void *payload, u32 payload_len, void *ctx); + void *payload, u32 payload_len, + s32 sender_instance_id, void *ctx); /* Opaque type for pending message tracking */ struct mk_pending_msg; @@ -779,6 +781,9 @@ struct mk_instance { /* CPU resources */ struct mk_cpu_set *cpus; /* Set of assigned physical CPU IDs */ + /* Pins the CPU selected for control messages and forwarded IRQs. */ + struct rw_semaphore control_route_sem; + mk_phys_cpu_t irq_route_cpu; /* PCI device resources */ struct list_head pci_devices; /* List of struct mk_pci_device */ @@ -866,6 +871,20 @@ struct mk_instance { struct kref refcount; /* Reference count for cleanup */ }; +static inline mk_phys_cpu_t +mk_instance_irq_route_load(const struct mk_instance *instance) +{ + /* Pair with the release store that publishes route changes. */ + return smp_load_acquire(&instance->irq_route_cpu); +} + +static inline void mk_instance_irq_route_store(struct mk_instance *instance, + mk_phys_cpu_t target) +{ + /* Publish the route after its associated control state. */ + smp_store_release(&instance->irq_route_cpu, target); +} + /** * Device Tree Parsing Functions */ @@ -1069,6 +1088,7 @@ struct mk_instance *mk_instance_find(int mk_id); void mk_instance_put(struct mk_instance *instance); void mk_instance_set_state(struct mk_instance *instance, enum mk_instance_state state); +void mk_instance_mark_failed(struct mk_instance *instance); int mk_instance_abort_spawn(struct mk_instance *instance); /* Kimage-based access to the instance memory pool */ diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index 7272f82400e8ed..1907c2a8e86221 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -595,6 +595,7 @@ void kimage_free(struct kimage *image) { kimage_entry_t *ptr, entry; kimage_entry_t ind = 0; + struct mk_instance *route_instance = NULL; if (!image) return; @@ -609,6 +610,10 @@ void kimage_free(struct kimage *image) if (image->type == KEXEC_TYPE_MULTIKERNEL) { unsigned long i; + route_instance = image->mk_instance; + if (route_instance) + down_write(&route_instance->control_route_sem); + /* Stop delivery before image-owned shared pages are returned. */ #ifdef CONFIG_MULTIKERNEL if (image->mk_instance) @@ -635,7 +640,6 @@ void kimage_free(struct kimage *image) image->mk_instance->ipi_phys = 0; image->mk_instance->kimage = NULL; mk_instance_set_state(image->mk_instance, MK_STATE_READY); - mk_instance_put(image->mk_instance); image->mk_instance = NULL; } @@ -651,6 +655,10 @@ void kimage_free(struct kimage *image) __free_pages(phys_to_page(image->mk_ipi), order); image->mk_ipi = 0; } + if (route_instance) { + up_write(&route_instance->control_route_sem); + mk_instance_put(route_instance); + } } #ifdef CONFIG_CRASH_DUMP if (image->vmcoreinfo_data_copy) { @@ -1693,6 +1701,8 @@ int multikernel_kexec_by_id(int mk_id) { struct kimage *mk_image; struct mk_instance *instance; + bool transaction_locked = false; + bool route_locked = false; int cpu = -1; int i, rc; @@ -1713,9 +1723,16 @@ int multikernel_kexec_by_id(int mk_id) rc = -EINVAL; goto unlock; } + mk_cpu_transaction_lock(); + transaction_locked = true; + down_write(&instance->control_route_sem); + route_locked = true; if (!mk_cpu_set_empty(instance->cpus)) { mk_phys_cpu_t phys_cpu = mk_cpu_set_first(instance->cpus); + if (!mk_cpu_set_contains(instance->cpus, + mk_instance_irq_route_load(instance))) + mk_instance_irq_route_store(instance, phys_cpu); cpu = arch_cpu_from_physical_id(phys_cpu); if (cpu < 0) { pr_err("Physical CPU %llu not found in logical CPU map\n", phys_cpu); @@ -1819,7 +1836,13 @@ int multikernel_kexec_by_id(int mk_id) */ rc = mk_instance_set_kexec_active(mk_image->mk_id); if (rc) { - int abort_ret = mk_instance_abort_spawn(instance); + int abort_ret; + + up_write(&instance->control_route_sem); + route_locked = false; + mk_cpu_transaction_unlock(); + transaction_locked = false; + abort_ret = mk_instance_abort_spawn(instance); if (abort_ret) pr_crit("Instance %d activation abort failed: %d\n", @@ -1827,10 +1850,16 @@ int multikernel_kexec_by_id(int mk_id) goto unlock; } + up_write(&instance->control_route_sem); + mk_cpu_transaction_unlock(); kexec_unlock(); return 0; unlock: + if (route_locked) + up_write(&instance->control_route_sem); + if (transaction_locked) + mk_cpu_transaction_unlock(); kexec_unlock(); return rc; } diff --git a/kernel/multikernel/core.c b/kernel/multikernel/core.c index 427b7d96f6aa1c..ea69e3769c626a 100644 --- a/kernel/multikernel/core.c +++ b/kernel/multikernel/core.c @@ -16,7 +16,7 @@ #include #include "internal.h" -/* CPU moves hold the transaction lock before the ownership lock. */ +/* Lock order: transaction -> route write -> ownership -> resources. */ static DEFINE_MUTEX(mk_cpu_transaction_mutex); static DEFINE_MUTEX(mk_cpu_ownership_mutex); @@ -45,6 +45,55 @@ void mk_cpu_ownership_assert_held(void) lockdep_assert_held(&mk_cpu_ownership_mutex); } +static int __mk_instance_migrate_irq_route(struct mk_instance *instance, + const struct mk_cpu_set *removing) +{ + mk_phys_cpu_t replacement = MK_PHYS_CPU_INVALID; + mk_phys_cpu_t route_cpu; + mk_phys_cpu_t phys_cpu; + unsigned int i; + + lockdep_assert_held_write(&instance->control_route_sem); + route_cpu = mk_instance_irq_route_load(instance); + if (route_cpu == MK_PHYS_CPU_INVALID || + !mk_cpu_set_contains(removing, route_cpu)) + return 0; + + mk_cpu_ownership_lock(); + mk_cpu_set_for_each(i, phys_cpu, instance->cpus) { + if (!mk_cpu_set_contains(removing, phys_cpu)) { + replacement = phys_cpu; + break; + } + } + mk_cpu_ownership_unlock(); + if (replacement == MK_PHYS_CPU_INVALID && + READ_ONCE(instance->state) == MK_STATE_ACTIVE) + return -EBUSY; + + mutex_lock(&instance->resource_mutex); + if (replacement != MK_PHYS_CPU_INVALID) + mk_instance_irq_route_store(instance, replacement); + mk_pci_sync_instance_irq_route(instance); + if (replacement == MK_PHYS_CPU_INVALID) + mk_instance_irq_route_store(instance, MK_PHYS_CPU_INVALID); + mutex_unlock(&instance->resource_mutex); + return 0; +} + +int mk_instance_migrate_irq_route(struct mk_instance *instance, + const struct mk_cpu_set *removing) +{ + int ret; + + if (!instance || !removing) + return -EINVAL; + down_write(&instance->control_route_sem); + ret = __mk_instance_migrate_irq_route(instance, removing); + up_write(&instance->control_route_sem); + return ret; +} + static void mk_instance_return_all_cpus(struct mk_instance *instance) { if (!instance || mk_cpu_set_empty(instance->cpus)) @@ -270,6 +319,8 @@ struct mk_instance *mk_instance_alloc(int id, const char *name) goto err_free_name; instance->state = MK_STATE_READY; + init_rwsem(&instance->control_route_sem); + instance->irq_route_cpu = MK_PHYS_CPU_INVALID; instance->ipi_target = MK_PHYS_CPU_INVALID; raw_spin_lock_init(&instance->ipi_endpoint.tx_lock); raw_spin_lock_init(&instance->ipi_endpoint.rx_lock); @@ -373,6 +424,19 @@ void mk_instance_set_state(struct mk_instance *instance, */ } +void mk_instance_mark_failed(struct mk_instance *instance) +{ + if (!instance) + return; + mk_cpu_transaction_lock(); + down_write(&instance->control_route_sem); + mutex_lock(&instance->resource_mutex); + mk_instance_set_state(instance, MK_STATE_FAILED); + mutex_unlock(&instance->resource_mutex); + up_write(&instance->control_route_sem); + mk_cpu_transaction_unlock(); +} + struct mk_instance *mk_instance_find_by_name(const char *name) { struct mk_instance *instance; @@ -530,6 +594,7 @@ int mk_instance_transfer_cpus(struct mk_instance *instance, return -ENOMEM; mk_cpu_transaction_lock(); + down_write(&instance->control_route_sem); mk_cpu_ownership_lock(); ret = mk_cpu_set_copy(requested, cpus); if (ret) @@ -557,6 +622,9 @@ int mk_instance_transfer_cpus(struct mk_instance *instance, unavailable++; } } + if (mk_instance_irq_route_load(instance) == MK_PHYS_CPU_INVALID) + mk_instance_irq_route_store(instance, + mk_cpu_set_first(instance->cpus)); if (unavailable > 0) { pr_err("Instance %d (%s): %d CPUs are not available\n", @@ -581,6 +649,7 @@ int mk_instance_transfer_cpus(struct mk_instance *instance, ret = 0; out_unlock: mk_cpu_ownership_unlock(); + up_write(&instance->control_route_sem); mk_cpu_transaction_unlock(); mk_cpu_set_free(requested); return ret; @@ -616,6 +685,7 @@ int mk_instance_return_cpus(struct mk_instance *instance, return -ENOMEM; mk_cpu_transaction_lock(); + down_write(&instance->control_route_sem); mk_cpu_ownership_lock(); ret = mk_cpu_set_copy(requested, cpus); if (ret) @@ -650,6 +720,11 @@ int mk_instance_return_cpus(struct mk_instance *instance, goto out_unlock; mk_cpu_set_format(buf, sizeof(buf), requested); + mk_cpu_ownership_unlock(); + ret = __mk_instance_migrate_irq_route(instance, requested); + if (ret) + goto out_route; + mk_cpu_ownership_lock(); mk_cpu_set_for_each(i, phys_cpu, requested) { mk_cpu_set_add(mk_pool->cpus, phys_cpu); @@ -662,6 +737,8 @@ int mk_instance_return_cpus(struct mk_instance *instance, ret = 0; out_unlock: mk_cpu_ownership_unlock(); +out_route: + up_write(&instance->control_route_sem); mk_cpu_transaction_unlock(); mk_cpu_set_free(requested); return ret; @@ -1520,7 +1597,14 @@ static void mk_instance_settle_halted(struct mk_instance *instance) { pr_info("Instance %d (%s) halted, CPUs parking in pool\n", instance->id, instance->name); + mk_cpu_transaction_lock(); + down_write(&instance->control_route_sem); + mutex_lock(&instance->resource_mutex); + mk_instance_irq_route_store(instance, MK_PHYS_CPU_INVALID); mk_instance_set_state(instance, MK_STATE_LOADED); + mutex_unlock(&instance->resource_mutex); + up_write(&instance->control_route_sem); + mk_cpu_transaction_unlock(); } struct mk_halted_work { @@ -1547,7 +1631,8 @@ static void mk_halted_work_fn(struct work_struct *work) } static void mk_system_msg_handler(u32 msg_type, u32 subtype, - void *payload, u32 payload_len, void *ctx) + void *payload, u32 payload_len, + s32 sender_instance_id, void *ctx) { if (msg_type != MK_MSG_SYSTEM) return; @@ -1560,7 +1645,7 @@ static void mk_system_msg_handler(u32 msg_type, u32 subtype, if (payload_len < sizeof(*req)) return; - pr_info("Shutdown requested by instance %d\n", req->sender_instance_id); + pr_info("Shutdown requested by instance %d\n", sender_instance_id); sw = kmalloc(sizeof(*sw), GFP_ATOMIC); if (!sw) @@ -1568,7 +1653,7 @@ static void mk_system_msg_handler(u32 msg_type, u32 subtype, INIT_WORK(&sw->work, mk_shutdown_work_fn); sw->flags = req->flags; - sw->sender_instance_id = req->sender_instance_id; + sw->sender_instance_id = sender_instance_id; schedule_work(&sw->work); break; } @@ -1577,6 +1662,8 @@ static void mk_system_msg_handler(u32 msg_type, u32 subtype, if (payload_len < sizeof(*ack)) return; + if (ack->resource_id != sender_instance_id) + return; /* * Reply to a shutdown this kernel requested: wake the * requester, which waits for the instance's CPUs to park @@ -1592,6 +1679,8 @@ static void mk_system_msg_handler(u32 msg_type, u32 subtype, if (payload_len < sizeof(*ack)) return; + if (ack->resource_id != sender_instance_id) + return; /* * The instance halted itself; nobody is waiting on it, so @@ -1892,7 +1981,7 @@ int mk_instance_abort_spawn(struct mk_instance *instance) mk_ipi_endpoint_close(instance); ret = __mk_instance_force_halt(instance, true); if (ret && instance) - mk_instance_set_state(instance, MK_STATE_FAILED); + mk_instance_mark_failed(instance); return ret; } diff --git a/kernel/multikernel/hotplug.c b/kernel/multikernel/hotplug.c index d91ea445fe5298..7f529d788748ac 100644 --- a/kernel/multikernel/hotplug.c +++ b/kernel/multikernel/hotplug.c @@ -989,7 +989,8 @@ static int mk_handle_device_remove(struct mk_device_resource_payload *payload, u * message subtype. */ static void mk_resource_msg_handler(u32 msg_type, u32 subtype, - void *payload, u32 payload_len, void *ctx) + void *payload, u32 payload_len, + s32 sender_instance_id, void *ctx) { int ret = 0; @@ -1251,6 +1252,7 @@ int mk_pool_device_remove(u16 domain, u8 bus, u8 devfn, */ int mk_send_cpu_remove(int instance_id, mk_phys_cpu_t cpu_id) { + struct mk_cpu_set removing = { .nr = 1, .cap = 1, .ids = &cpu_id }; struct mk_cpu_resource_payload payload = { .cpu_id = cpu_id, .numa_node = 0, @@ -1259,8 +1261,11 @@ int mk_send_cpu_remove(int instance_id, mk_phys_cpu_t cpu_id) }; struct mk_pending_msg *pending; struct mk_instance *target_instance; + mk_phys_cpu_t route_cpu; int ret; + raw_spin_lock_init(&removing.lock); + /* For self-removal, execute directly (we're in process context) */ if (instance_id == mk_self->id) { if (mk_pool) @@ -1313,20 +1318,29 @@ int mk_send_cpu_remove(int instance_id, mk_phys_cpu_t cpu_id) if (!mk_cpu_set_contains(target_instance->cpus, cpu_id)) { pr_err("Multikernel hotplug: CPU %llu not assigned to instance %d\n", cpu_id, instance_id); - mk_msg_pending_wait(pending, 0); ret = -EINVAL; - goto unlock_ownership; + mk_cpu_ownership_unlock(); + mk_msg_pending_wait(pending, 0); + goto unlock_transaction; } ret = mk_cpu_set_reserve(mk_pool->cpus, 1); if (ret) { + mk_cpu_ownership_unlock(); mk_msg_pending_wait(pending, 0); - goto unlock_ownership; + goto unlock_transaction; } mk_cpu_ownership_unlock(); + ret = mk_instance_migrate_irq_route(target_instance, &removing); + if (ret) { + mk_msg_pending_wait(pending, 0); + goto unlock_transaction; + } + route_cpu = mk_instance_irq_route_load(target_instance); - ret = mk_send_message(target_instance->id, MK_MSG_RESOURCE, - MK_RES_CPU_REMOVE, &payload, sizeof(payload)); + ret = mk_send_message_to_cpu(target_instance, route_cpu, + MK_MSG_RESOURCE, MK_RES_CPU_REMOVE, + &payload, sizeof(payload)); if (ret < 0) { mk_msg_pending_wait(pending, 0); /* Immediate cleanup */ goto unlock_transaction; @@ -1351,6 +1365,7 @@ int mk_send_cpu_remove(int instance_id, mk_phys_cpu_t cpu_id) goto unlock_transaction; } + down_write(&target_instance->control_route_sem); mk_cpu_ownership_lock(); if (!mk_cpu_set_contains(target_instance->cpus, cpu_id)) { ret = -ESTALE; @@ -1367,6 +1382,7 @@ int mk_send_cpu_remove(int instance_id, mk_phys_cpu_t cpu_id) ret = 0; unlock_ownership: mk_cpu_ownership_unlock(); + up_write(&target_instance->control_route_sem); unlock_transaction: mk_cpu_transaction_unlock(); out: @@ -1440,12 +1456,15 @@ int mk_send_cpu_add(int instance_id, mk_phys_cpu_t cpu_id, u32 numa_node, u32 fl pr_err("Multikernel hotplug: CPU %llu not available in the pool\n", cpu_id); ret = -EBUSY; - goto unlock_ownership; + mk_cpu_ownership_unlock(); + goto unlock_transaction; } ret = mk_cpu_set_reserve(target_instance->cpus, 1); - if (ret) - goto unlock_ownership; + if (ret) { + mk_cpu_ownership_unlock(); + goto unlock_transaction; + } mk_cpu_ownership_unlock(); /* @@ -1467,8 +1486,9 @@ int mk_send_cpu_add(int instance_id, mk_phys_cpu_t cpu_id, u32 numa_node, u32 fl goto unlock_transaction; } - ret = mk_send_message(target_instance->id, MK_MSG_RESOURCE, - MK_RES_CPU_ADD, &payload, sizeof(payload)); + ret = mk_send_message_to_instance(target_instance, MK_MSG_RESOURCE, + MK_RES_CPU_ADD, &payload, + sizeof(payload)); if (ret < 0) { mk_msg_pending_wait(pending, 0); /* Immediate cleanup */ mk_repark_cpu_to_host(target_instance, cpu_id); @@ -1487,6 +1507,7 @@ int mk_send_cpu_add(int instance_id, mk_phys_cpu_t cpu_id, u32 numa_node, u32 fl goto unlock_transaction; } + down_write(&target_instance->control_route_sem); mk_cpu_ownership_lock(); if (!mk_cpu_set_contains(mk_pool->cpus, cpu_id)) { ret = -ESTALE; @@ -1499,10 +1520,13 @@ int mk_send_cpu_add(int instance_id, mk_phys_cpu_t cpu_id, u32 numa_node, u32 fl goto unlock_ownership; } mk_cpu_set_del(mk_pool->cpus, cpu_id); + if (mk_instance_irq_route_load(target_instance) == MK_PHYS_CPU_INVALID) + mk_instance_irq_route_store(target_instance, cpu_id); ret = 0; unlock_ownership: mk_cpu_ownership_unlock(); + up_write(&target_instance->control_route_sem); unlock_transaction: mk_cpu_transaction_unlock(); out: diff --git a/kernel/multikernel/instance_dt.c b/kernel/multikernel/instance_dt.c index 85627764afab4c..06517def8e3e0d 100644 --- a/kernel/multikernel/instance_dt.c +++ b/kernel/multikernel/instance_dt.c @@ -370,6 +370,7 @@ static int __init mk_restore_host_instance(void) if (!hi) return -ENOMEM; hi->ipi_target = parent_cpu; + mk_instance_irq_route_store(hi, parent_cpu); ret = mk_cpu_set_add(hi->cpus, parent_cpu); if (ret) goto err_free; diff --git a/kernel/multikernel/internal.h b/kernel/multikernel/internal.h index 26c3532e3a244d..30b733b11be10c 100644 --- a/kernel/multikernel/internal.h +++ b/kernel/multikernel/internal.h @@ -23,8 +23,18 @@ int mk_instance_force_halt(struct mk_instance *instance); int mk_send_ipi_data(struct mk_instance *instance, void *data, size_t data_size, unsigned long type); struct mk_shared_data *mk_instance_halt_data(struct mk_instance *instance); +int mk_send_ipi_data_to_cpu(struct mk_instance *instance, + mk_phys_cpu_t target, void *data, + size_t data_size, unsigned long type); void mk_poll_ipi_messages(void); +/* messaging.c */ +int mk_send_message_to_instance(struct mk_instance *instance, u32 msg_type, + u32 subtype, void *payload, u32 payload_len); +int mk_send_message_to_cpu(struct mk_instance *instance, + mk_phys_cpu_t target, u32 msg_type, u32 subtype, + void *payload, u32 payload_len); + /* kernfs.c */ extern struct kernfs_node *mk_root_kn; extern struct kernfs_node *mk_instances_kn; @@ -33,6 +43,12 @@ int mk_create_instance_from_dtb(const char *name, int id, const void *fdt, struct mk_instance *mk_instance_find_by_name(const char *name); int mk_instance_destroy(struct mk_instance *instance); int mk_instance_release_resources(struct mk_instance *instance); +void mk_cpu_ownership_lock(void); +void mk_cpu_ownership_unlock(void); +void mk_cpu_ownership_assert_held(void); +/* Caller serializes CPU ownership changes with mk_cpu_transaction_lock(). */ +int mk_instance_migrate_irq_route(struct mk_instance *instance, + const struct mk_cpu_set *removing); /* dts.c */ int mk_dt_parse_chosen(const void *fdt, int chosen_node, @@ -47,11 +63,6 @@ int mk_dt_generate_instance_dtb(struct mk_instance *instance, int mk_pci_parse_bdf(const char *pci_id, int len, u16 *domain, u8 *bus, u8 *slot, u8 *func); -/* CPU ownership serialization: transaction must be acquired first. */ -void mk_cpu_ownership_lock(void); -void mk_cpu_ownership_unlock(void); -void mk_cpu_ownership_assert_held(void); - /* pci.c */ int mk_pci_lease_system_init(void); void mk_pci_lease_system_cleanup(void); @@ -65,7 +76,11 @@ int mk_pci_assign_device(struct mk_instance *instance, u16 domain, u8 bus, int mk_pci_unassign_device(struct mk_instance *instance, u16 domain, u8 bus, u8 devfn); int mk_pci_release_assignments(struct mk_instance *instance); -int mk_instance_force_halt(struct mk_instance *instance); +static inline unsigned int +mk_pci_sync_instance_irq_route(struct mk_instance *instance) +{ + return 0; +} /* overlay.c */ extern struct kernfs_node *mk_overlay_root_kn; extern struct mutex mk_overlay_mutex; diff --git a/kernel/multikernel/ipi.c b/kernel/multikernel/ipi.c index c069b1a4918edd..ca7afaf3a7439d 100644 --- a/kernel/multikernel/ipi.c +++ b/kernel/multikernel/ipi.c @@ -396,10 +396,11 @@ int mk_reply_publish(struct mk_instance *instance, if (!instance || !reply || reply->slot >= MK_REPLY_SLOTS) return -EINVAL; + down_read(&instance->control_route_sem); shared = instance->ipi_data; ret = shared ? 0 : -ENODEV; if (ret) - return ret; + goto unlock_route; slot = &shared->replies.slots[reply->slot]; writing = mk_reply_token(reply->generation, MK_REPLY_WRITING); executing = mk_reply_token(reply->generation, MK_REPLY_EXECUTING); @@ -417,25 +418,31 @@ int mk_reply_publish(struct mk_instance *instance, if (old == abandoned) { atomic64_set_release(&slot->state_generation, free); atomic_inc(&shared->replies.late_replies); - return -ESTALE; + ret = -ESTALE; + goto unlock_route; } if (old == committed) { atomic64_set_release(&slot->state_generation, free); atomic_inc(&shared->replies.late_replies); - return -ESTALE; + ret = -ESTALE; + goto unlock_route; } if (old != executing && old != writing) { atomic_inc(&shared->replies.late_replies); - return -EIO; + ret = -EIO; + goto unlock_route; } - target = READ_ONCE(instance->ipi_target); - if (target == MK_PHYS_CPU_INVALID) - target = mk_cpu_set_first(instance->cpus); - if (target == MK_PHYS_CPU_INVALID) - return -ENODEV; + target = mk_instance_irq_route_load(instance); + if (target == MK_PHYS_CPU_INVALID) { + ret = -ENODEV; + goto unlock_route; + } + ret = 0; mk_arch_send_ipi(target); - return 0; +unlock_route: + up_read(&instance->control_route_sem); + return ret; } void mk_reply_scan(struct mk_shared_data *shared) @@ -508,18 +515,19 @@ int mk_arm_force_halt(struct mk_instance *instance) return 0; } -int mk_send_ipi_data(struct mk_instance *instance, void *data, - size_t data_size, unsigned long type) +static int __mk_send_ipi_data(struct mk_instance *instance, + mk_phys_cpu_t target, void *data, + size_t data_size, unsigned long type) { struct mk_ipi_endpoint *endpoint; struct mk_ipi_data *slot; struct mk_shared_data *shared; - mk_phys_cpu_t target; unsigned long flags; u32 idx; int ret = 0; - if (!instance || data_size > MK_MAX_DATA_SIZE || (data_size && !data)) + if (!instance || target == MK_PHYS_CPU_INVALID || + data_size > MK_MAX_DATA_SIZE || (data_size && !data)) return -EINVAL; endpoint = &instance->ipi_endpoint; if (!READ_ONCE(endpoint->registered)) @@ -534,12 +542,6 @@ int mk_send_ipi_data(struct mk_instance *instance, void *data, ret = -ENODEV; goto unlock; } - target = endpoint->parent_side ? mk_cpu_set_first(instance->cpus) : - READ_ONCE(shared->parent_doorbell_cpu); - if (target == MK_PHYS_CPU_INVALID) { - ret = -ENODEV; - goto unlock; - } if (endpoint->parent_side) WRITE_ONCE(shared->child_doorbell_cpu, target); idx = endpoint->tx_head & (MK_IPI_RING_SIZE - 1); @@ -568,6 +570,28 @@ int mk_send_ipi_data(struct mk_instance *instance, void *data, return ret; } +int mk_send_ipi_data_to_cpu(struct mk_instance *instance, + mk_phys_cpu_t target, void *data, + size_t data_size, unsigned long type) +{ + return __mk_send_ipi_data(instance, target, data, data_size, type); +} + +int mk_send_ipi_data(struct mk_instance *instance, void *data, + size_t data_size, unsigned long type) +{ + mk_phys_cpu_t target; + int ret; + + if (!instance) + return -EINVAL; + down_read(&instance->control_route_sem); + target = mk_instance_irq_route_load(instance); + ret = __mk_send_ipi_data(instance, target, data, data_size, type); + up_read(&instance->control_route_sem); + return ret; +} + int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, unsigned long type) { diff --git a/kernel/multikernel/messaging.c b/kernel/multikernel/messaging.c index 2cd2b7682f8d3e..fbd5682d2dd7d3 100644 --- a/kernel/multikernel/messaging.c +++ b/kernel/multikernel/messaging.c @@ -11,6 +11,7 @@ #include #include #include +#include "internal.h" /* Pending message tracking for request-response pattern */ struct mk_pending_msg { @@ -86,7 +87,9 @@ static void mk_message_type_ipi_callback(struct mk_ipi_data *data, void *ctx) msg_type, msg_subtype, payload_len, data->sender_cpu); /* Call the registered handler for this message type */ - type_handler->msg_handler(msg_type, msg_subtype, payload, payload_len, type_handler->context); + type_handler->msg_handler(msg_type, msg_subtype, payload, payload_len, + data->sender_instance_id, + type_handler->context); } /* @@ -190,8 +193,10 @@ int mk_msg_pending_wait(struct mk_pending_msg *pending, unsigned long timeout_ms * * Returns 0 on success, negative error code on failure */ -int mk_send_message(int instance_id, u32 msg_type, u32 subtype, - void *payload, u32 payload_len) +static int __mk_send_message(struct mk_instance *instance, int instance_id, + mk_phys_cpu_t target, + u32 msg_type, u32 subtype, void *payload, + u32 payload_len) { struct mk_message *msg; size_t total_size; @@ -223,7 +228,14 @@ int mk_send_message(int instance_id, u32 msg_type, u32 subtype, memcpy(msg->payload, payload, payload_len); /* Send via IPI using the message type as IPI type */ - ret = multikernel_send_ipi_data(instance_id, msg, total_size, msg_type); + if (instance && target != MK_PHYS_CPU_INVALID) + ret = mk_send_ipi_data_to_cpu(instance, target, msg, total_size, + msg_type); + else if (instance) + ret = mk_send_ipi_data(instance, msg, total_size, msg_type); + else + ret = multikernel_send_ipi_data(instance_id, msg, total_size, + msg_type); /* Clean up temporary buffer */ kfree(msg); @@ -238,6 +250,32 @@ int mk_send_message(int instance_id, u32 msg_type, u32 subtype, return 0; } + +int mk_send_message_to_instance(struct mk_instance *instance, u32 msg_type, + u32 subtype, void *payload, u32 payload_len) +{ + if (!instance) + return -EINVAL; + return __mk_send_message(instance, instance->id, MK_PHYS_CPU_INVALID, + msg_type, subtype, payload, payload_len); +} + +int mk_send_message_to_cpu(struct mk_instance *instance, + mk_phys_cpu_t target, u32 msg_type, u32 subtype, + void *payload, u32 payload_len) +{ + if (!instance || target == MK_PHYS_CPU_INVALID) + return -EINVAL; + return __mk_send_message(instance, instance->id, target, msg_type, + subtype, payload, payload_len); +} + +int mk_send_message(int instance_id, u32 msg_type, u32 subtype, + void *payload, u32 payload_len) +{ + return __mk_send_message(NULL, instance_id, MK_PHYS_CPU_INVALID, + msg_type, subtype, payload, payload_len); +} EXPORT_SYMBOL(mk_send_message); /** diff --git a/net/vmw_vsock/mk_transport.c b/net/vmw_vsock/mk_transport.c index 84961b2f355aa2..086e1b32702d35 100644 --- a/net/vmw_vsock/mk_transport.c +++ b/net/vmw_vsock/mk_transport.c @@ -297,7 +297,8 @@ static void mk_vsock_rx_work(struct work_struct *work) } static void mk_vsock_ipi_handler(u32 msg_type, u32 subtype, - void *payload, u32 payload_len, void *ctx) + void *payload, u32 payload_len, + s32 sender_instance_id, void *ctx) { struct sk_buff *skb; struct virtio_vsock_hdr *hdr; From f73c64e733a95fec8a90b93b55f3c62bec9a82bd Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Tue, 4 Aug 2026 11:04:33 +0300 Subject: [PATCH 13/24] multikernel: mediate PCI config with direct replies Proxy spawn PCI configuration through the host so only a live VF lease selected by an assigned BDF can access hardware. Return results through preallocated generation-tagged reply slots. Atomic callers wait only on their slot with a bounded deadline and never drain the general ring. Bump the exact transport ABI to version 4. Signed-off-by: Nikolay Nikolaev --- arch/x86/multikernel/pci.c | 156 ++++++++++++------ include/linux/multikernel.h | 27 +++ kernel/multikernel/internal.h | 3 + kernel/multikernel/ipi.c | 30 +++- kernel/multikernel/pci.c | 300 ++++++++++++++++++++++++++++++++-- 5 files changed, 438 insertions(+), 78 deletions(-) diff --git a/arch/x86/multikernel/pci.c b/arch/x86/multikernel/pci.c index 2a3d9a84375386..7ebbfa45e762fa 100644 --- a/arch/x86/multikernel/pci.c +++ b/arch/x86/multikernel/pci.c @@ -6,6 +6,7 @@ * filtered here and become host-mediated once the RPC transport is installed. */ #include +#include #include #include #include @@ -14,9 +15,89 @@ #include #include -static const struct pci_raw_ops *mk_pci_native_raw_ops; -static const struct pci_raw_ops *mk_pci_native_raw_ext_ops; static bool mk_pci_roots_ready; +static atomic64_t mk_pci_request_id = ATOMIC64_INIT(0); +static atomic64_t mk_pci_cfg_count = ATOMIC64_INIT(0); +static atomic64_t mk_pci_cfg_total_ns = ATOMIC64_INIT(0); +static atomic64_t mk_pci_cfg_max_ns = ATOMIC64_INIT(0); + +static void mk_pci_record_latency(u64 start) +{ + u64 elapsed = ktime_get_mono_fast_ns() - start; + u64 old_max = atomic64_read(&mk_pci_cfg_max_ns); + + atomic64_inc(&mk_pci_cfg_count); + atomic64_add(elapsed, &mk_pci_cfg_total_ns); + while (elapsed > old_max) { + u64 previous = atomic64_cmpxchg(&mk_pci_cfg_max_ns, old_max, + elapsed); + + if (previous == old_max) + break; + old_max = previous; + } +} + +static int mk_pci_remote_config(unsigned int domain, unsigned int bus, + unsigned int devfn, int where, int size, + bool write, u32 *value) +{ + struct mk_pci_cfg_request request = { + .request_id = atomic64_inc_return(&mk_pci_request_id), + .lifecycle_epoch = mk_self && mk_self->ipi_data ? + READ_ONCE(mk_self->ipi_data->spawn_epoch) : 0, + .sender_instance_id = mk_self ? mk_self->id : -1, + .domain = domain, + .bus = bus, + .devfn = devfn, + .reg = where, + .len = size, + .write = write, + .value = *value, + }; + struct mk_reply_handle reply; + s32 status; + u32 response_value; + u64 start = ktime_get_mono_fast_ns(); + int ret; + + if (!mk_self || !mk_self->ipi_data || !host_instance) { + ret = -ENODEV; + goto out_error; + } + ret = mk_reply_reserve(mk_self->ipi_data, MK_REPLY_PCI_CFG, + request.request_id, &reply); + if (ret) + goto out_error; + request.reply_slot = reply.slot; + request.reply_generation = reply.generation; + ret = mk_send_message(host_instance->id, MK_MSG_PCI, MK_PCI_CFG_REQUEST, + &request, sizeof(request)); + if (ret) { + mk_reply_release(mk_self->ipi_data, &reply); + goto out_error; + } + ret = mk_reply_wait_atomic(mk_self->ipi_data, &reply, 20000, + &status, &response_value); + if (ret) + goto out_error; + if (status < 0) { + ret = status; + goto out_error; + } + if (!write) + *value = response_value; + mk_pci_record_latency(start); + return status; + +out_error: + if (ret < 0) { + pr_err_ratelimited("Multikernel PCI config request timed out or failed to send: %d\n", + ret); + return PCIBIOS_SET_FAILED; + } + return PCIBIOS_SET_FAILED; +} static bool mk_pci_identity_read(u16 vendor, u16 device, int where, int size, u32 *value) @@ -32,8 +113,7 @@ static bool mk_pci_identity_read(u16 vendor, u16 device, int where, int size, return true; } -static int mk_pci_raw_read(const struct pci_raw_ops *native, - unsigned int domain, unsigned int bus, +static int mk_pci_raw_read(unsigned int domain, unsigned int bus, unsigned int devfn, int where, int size, u32 *value) { @@ -46,65 +126,30 @@ static int mk_pci_raw_read(const struct pci_raw_ops *native, } if (mk_pci_identity_read(vendor, device, where, size, value)) return PCIBIOS_SUCCESSFUL; - return native->read(domain, bus, devfn, where, size, value); + return mk_pci_remote_config(domain, bus, devfn, where, size, false, + value); } -static int mk_pci_raw_write(const struct pci_raw_ops *native, - unsigned int domain, unsigned int bus, +static int mk_pci_raw_write(unsigned int domain, unsigned int bus, unsigned int devfn, int where, int size, u32 value) { if (!mk_pci_get_assigned_identity_bdf(domain, bus, devfn, NULL, NULL)) return PCIBIOS_DEVICE_NOT_FOUND; - return native->write(domain, bus, devfn, where, size, value); -} - -static int mk_pci_read(unsigned int domain, unsigned int bus, - unsigned int devfn, int where, int size, u32 *value) -{ - return mk_pci_raw_read(mk_pci_native_raw_ops, domain, bus, devfn, - where, size, value); -} - -static int mk_pci_write(unsigned int domain, unsigned int bus, - unsigned int devfn, int where, int size, u32 value) -{ - return mk_pci_raw_write(mk_pci_native_raw_ops, domain, bus, devfn, - where, size, value); -} - -static int mk_pci_ext_read(unsigned int domain, unsigned int bus, - unsigned int devfn, int where, int size, u32 *value) -{ - return mk_pci_raw_read(mk_pci_native_raw_ext_ops, domain, bus, devfn, - where, size, value); -} - -static int mk_pci_ext_write(unsigned int domain, unsigned int bus, - unsigned int devfn, int where, int size, u32 value) -{ - return mk_pci_raw_write(mk_pci_native_raw_ext_ops, domain, bus, devfn, - where, size, value); + return mk_pci_remote_config(domain, bus, devfn, where, size, true, + &value); } static const struct pci_raw_ops mk_pci_filtered_raw_ops = { - .read = mk_pci_read, - .write = mk_pci_write, -}; - -static const struct pci_raw_ops mk_pci_filtered_raw_ext_ops = { - .read = mk_pci_ext_read, - .write = mk_pci_ext_write, + .read = mk_pci_raw_read, + .write = mk_pci_raw_write, }; static int __init x86_multikernel_pci_arch_init(void) { - if (!root_instance || !root_instance->pci_devices_valid) + if (!mk_self || !host_instance || !mk_self->pci_devices_valid) return 0; - - mk_pci_native_raw_ops = raw_pci_ops; - mk_pci_native_raw_ext_ops = raw_pci_ext_ops; raw_pci_ops = &mk_pci_filtered_raw_ops; - raw_pci_ext_ops = &mk_pci_filtered_raw_ext_ops; + raw_pci_ext_ops = &mk_pci_filtered_raw_ops; mk_pci_roots_ready = true; pr_notice("Multikernel selected filtered raw PCI config access\n"); return 0; @@ -163,14 +208,14 @@ static int __init x86_multikernel_pci_init(void) struct pci_bus *bus; struct pci_dev *pdev; - if (!root_instance) + if (!mk_self) panic("Multikernel lost restored instance metadata"); - if (!root_instance->pci_device_count) + if (!mk_self->pci_device_count) return 0; - if (!root_instance->pci_devices_valid || !mk_pci_roots_ready) + if (!mk_self->pci_devices_valid || !mk_pci_roots_ready) panic("Multikernel assigned PCI inventory is unavailable"); - list_for_each_entry(device, &root_instance->pci_devices, list) { + list_for_each_entry(device, &mk_self->pci_devices, list) { bus = mk_pci_get_root(device->domain, device->bus); if (IS_ERR(bus)) panic("Multikernel failed to create synthetic PCI root %04x:%02x: %ld", @@ -183,6 +228,13 @@ static int __init x86_multikernel_pci_init(void) device->func); pci_bus_add_devices(bus); } + if (atomic64_read(&mk_pci_cfg_count)) { + u64 count = atomic64_read(&mk_pci_cfg_count); + + pr_notice("Multikernel PCI control plane: %llu config requests, average %llu ns, max %llu ns\n", + count, atomic64_read(&mk_pci_cfg_total_ns) / count, + atomic64_read(&mk_pci_cfg_max_ns)); + } /* Suppress legacy bus 0 probing after every assigned function is present. */ return 0; diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index 1263d73612f8fb..a8413239d3a163 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -321,6 +321,7 @@ void mk_reply_scan(struct mk_shared_data *shared); #define MK_MSG_SYSTEM 0x3000 #define MK_MSG_USER 0x4000 #define MK_MSG_NETWORK 0x5000 +#define MK_MSG_PCI 0x6000 /* I/O interrupt forwarding subtypes */ #define MK_IO_IRQ_FORWARD (MK_MSG_IO + 1) @@ -348,6 +349,10 @@ void mk_reply_scan(struct mk_shared_data *shared); #define MK_NET_VSOCK_PKT (MK_MSG_NETWORK + 1) /* vsock packet */ #define MK_NET_DATA_READY (MK_MSG_NETWORK + 2) /* Data available notification */ +/* Host-mediated PCI control-plane subtypes */ +#define MK_PCI_CFG_REQUEST (MK_MSG_PCI + 1) +#define MK_PCI_CFG_RESPONSE (MK_MSG_PCI + 2) + /** * Core message structure */ @@ -371,6 +376,28 @@ struct mk_io_irq_payload { u32 flags; /* Control flags (priority, etc.) */ }; +struct mk_pci_cfg_request { + u64 request_id; + u64 lifecycle_epoch; + s32 sender_instance_id; + u16 domain; + u8 bus; + u8 devfn; + u16 reg; + u8 len; + u8 write; + u32 value; + u32 reply_slot; + u32 reply_reserved; + u64 reply_generation; +}; + +struct mk_pci_cfg_response { + u64 request_id; + s32 status; + u32 value; +}; + /* IRQ control flags */ #define MK_IRQ_HIGH_PRIORITY 0x01 #define MK_IRQ_LOW_LATENCY 0x02 diff --git a/kernel/multikernel/internal.h b/kernel/multikernel/internal.h index 30b733b11be10c..20420941ce4694 100644 --- a/kernel/multikernel/internal.h +++ b/kernel/multikernel/internal.h @@ -27,6 +27,9 @@ int mk_send_ipi_data_to_cpu(struct mk_instance *instance, mk_phys_cpu_t target, void *data, size_t data_size, unsigned long type); void mk_poll_ipi_messages(void); +int mk_reply_publish_route_locked(struct mk_instance *instance, + const struct mk_reply_handle *reply, + s32 status, u32 value); /* messaging.c */ int mk_send_message_to_instance(struct mk_instance *instance, u32 msg_type, diff --git a/kernel/multikernel/ipi.c b/kernel/multikernel/ipi.c index ca7afaf3a7439d..26336ff3eed90e 100644 --- a/kernel/multikernel/ipi.c +++ b/kernel/multikernel/ipi.c @@ -379,8 +379,9 @@ int mk_reply_begin_execute(struct mk_instance *instance, return -ESTALE; } -int mk_reply_publish(struct mk_instance *instance, - const struct mk_reply_handle *reply, s32 status, u32 value) +int mk_reply_publish_route_locked(struct mk_instance *instance, + const struct mk_reply_handle *reply, + s32 status, u32 value) { struct mk_shared_data *shared; struct mk_reply_slot *slot; @@ -396,11 +397,11 @@ int mk_reply_publish(struct mk_instance *instance, if (!instance || !reply || reply->slot >= MK_REPLY_SLOTS) return -EINVAL; - down_read(&instance->control_route_sem); + lockdep_assert_held_read(&instance->control_route_sem); shared = instance->ipi_data; ret = shared ? 0 : -ENODEV; if (ret) - goto unlock_route; + return ret; slot = &shared->replies.slots[reply->slot]; writing = mk_reply_token(reply->generation, MK_REPLY_WRITING); executing = mk_reply_token(reply->generation, MK_REPLY_EXECUTING); @@ -419,28 +420,39 @@ int mk_reply_publish(struct mk_instance *instance, atomic64_set_release(&slot->state_generation, free); atomic_inc(&shared->replies.late_replies); ret = -ESTALE; - goto unlock_route; + return ret; } if (old == committed) { atomic64_set_release(&slot->state_generation, free); atomic_inc(&shared->replies.late_replies); ret = -ESTALE; - goto unlock_route; + return ret; } if (old != executing && old != writing) { atomic_inc(&shared->replies.late_replies); ret = -EIO; - goto unlock_route; + return ret; } target = mk_instance_irq_route_load(instance); if (target == MK_PHYS_CPU_INVALID) { ret = -ENODEV; - goto unlock_route; + return ret; } ret = 0; mk_arch_send_ipi(target); -unlock_route: + return ret; +} + +int mk_reply_publish(struct mk_instance *instance, + const struct mk_reply_handle *reply, s32 status, u32 value) +{ + int ret; + + if (!instance) + return -EINVAL; + down_read(&instance->control_route_sem); + ret = mk_reply_publish_route_locked(instance, reply, status, value); up_read(&instance->control_route_sem); return ret; } diff --git a/kernel/multikernel/pci.c b/kernel/multikernel/pci.c index af68dce079c7c9..13e3cbe1830768 100644 --- a/kernel/multikernel/pci.c +++ b/kernel/multikernel/pci.c @@ -10,6 +10,8 @@ #include #include #include +#include +#include #include #include #include @@ -47,6 +49,68 @@ static DEFINE_MUTEX(mk_pci_lease_mutex); static DEFINE_SPINLOCK(mk_pci_active_lock); static LIST_HEAD(mk_pci_active_assignments); static bool mk_pci_notifier_registered; +static bool mk_pci_control_registered; +static mempool_t *mk_pci_control_pool; +static struct workqueue_struct *mk_pci_control_wq; +static DEFINE_SPINLOCK(mk_pci_control_lock); +static DECLARE_WAIT_QUEUE_HEAD(mk_pci_control_waitq); +static unsigned int mk_pci_control_active; +static bool mk_pci_control_shutdown = true; +static atomic64_t mk_pci_control_pool_exhausted = ATOMIC64_INIT(0); + +struct mk_pci_control_work { + struct work_struct work; + struct mk_pci_cfg_request request; + s32 sender_instance_id; +}; + +static bool mk_pci_control_handler_get(void) +{ + unsigned long flags; + bool acquired = false; + + spin_lock_irqsave(&mk_pci_control_lock, flags); + if (!mk_pci_control_shutdown) { + mk_pci_control_active++; + acquired = true; + } + spin_unlock_irqrestore(&mk_pci_control_lock, flags); + return acquired; +} + +static void mk_pci_control_handler_put(void) +{ + unsigned long flags; + bool drained; + + spin_lock_irqsave(&mk_pci_control_lock, flags); + WARN_ON_ONCE(!mk_pci_control_active); + if (mk_pci_control_active) + mk_pci_control_active--; + drained = !mk_pci_control_active; + spin_unlock_irqrestore(&mk_pci_control_lock, flags); + if (drained) + wake_up_all(&mk_pci_control_waitq); +} + +static void mk_pci_control_shutdown_begin(void) +{ + unsigned long flags; + + spin_lock_irqsave(&mk_pci_control_lock, flags); + mk_pci_control_shutdown = true; + spin_unlock_irqrestore(&mk_pci_control_lock, flags); +} + +static void mk_pci_control_shutdown_end(void) +{ + unsigned long flags; + + spin_lock_irqsave(&mk_pci_control_lock, flags); + mk_pci_control_active = 0; + mk_pci_control_shutdown = false; + spin_unlock_irqrestore(&mk_pci_control_lock, flags); +} static bool mk_pci_device_live(struct pci_dev *pdev) { @@ -90,7 +154,7 @@ mk_pci_find_root_inventory(const struct mk_pci_device *requested) { struct mk_pci_device *device; - list_for_each_entry(device, &root_instance->pci_devices, list) { + list_for_each_entry(device, &mk_self->pci_devices, list) { if (mk_pci_inventory_matches(device, requested)) return device; } @@ -101,7 +165,7 @@ mk_pci_find_root_inventory(const struct mk_pci_device *requested) static struct mk_pci_device * mk_pci_find_root_bdf(u16 domain, u8 bus, u8 devfn) { - return mk_pci_find_device_bdf(&root_instance->pci_devices, domain, bus, + return mk_pci_find_device_bdf(&mk_self->pci_devices, domain, bus, devfn); } @@ -121,6 +185,159 @@ mk_pci_find_assignment(struct mk_instance *instance, u16 domain, u8 bus, return NULL; } +static bool mk_pci_request_route_stale(struct mk_instance *instance, + s32 sender_instance_id, + u64 lifecycle_epoch) +{ + if (!instance || instance == mk_self || + instance->id != sender_instance_id) + return true; + if (READ_ONCE(instance->state) != MK_STATE_ACTIVE) + return true; + return !instance->ipi_data || !lifecycle_epoch || + READ_ONCE(instance->ipi_data->spawn_epoch) != lifecycle_epoch; +} + +static int mk_pci_config_access(struct mk_instance *instance, + const struct mk_pci_cfg_request *request, + u32 *value, + const struct mk_reply_handle *reply) +{ + struct mk_pci_assignment *assignment; + struct pci_dev *vf; + int ret; + + if (request->len != 1 && request->len != 2 && request->len != 4) + return PCIBIOS_BAD_REGISTER_NUMBER; + if (request->reg > PCI_CFG_SPACE_EXP_SIZE - request->len) + return PCIBIOS_BAD_REGISTER_NUMBER; + + mutex_lock(&instance->resource_mutex); + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + assignment = mk_pci_find_assignment(instance, request->domain, + request->bus, request->devfn); + if (!assignment || !assignment->assigned || + !mk_pci_device_live(assignment->vf)) { + ret = PCIBIOS_DEVICE_NOT_FOUND; + goto out; + } + ret = mk_reply_begin_execute(instance, reply); + if (ret) + goto out; + + vf = assignment->vf; + if (request->write) { + switch (request->len) { + case 1: + ret = pci_write_config_byte(vf, request->reg, + request->value); + break; + case 2: + ret = pci_write_config_word(vf, request->reg, + request->value); + break; + default: + ret = pci_write_config_dword(vf, request->reg, + request->value); + break; + } + } else { + switch (request->len) { + case 1: { + u8 data; + + ret = pci_read_config_byte(vf, request->reg, &data); + *value = data; + break; + } + case 2: { + u16 data; + + ret = pci_read_config_word(vf, request->reg, &data); + *value = data; + break; + } + default: + ret = pci_read_config_dword(vf, request->reg, value); + break; + } + } +out: + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + mutex_unlock(&instance->resource_mutex); + return ret; +} + +static void mk_pci_cfg_work_fn(struct work_struct *work) +{ + struct mk_pci_control_work *control_work = + container_of(work, struct mk_pci_control_work, work); + struct mk_reply_handle reply = { + .slot = control_work->request.reply_slot, + .kind = MK_REPLY_PCI_CFG, + .request_id = control_work->request.request_id, + .generation = control_work->request.reply_generation, + }; + struct mk_instance *instance; + u32 value = ~0U; + s32 status; + + instance = mk_instance_find(control_work->sender_instance_id); + if (!instance) + goto out; + down_read(&instance->control_route_sem); + status = mk_pci_request_route_stale(instance, + control_work->sender_instance_id, + control_work->request.lifecycle_epoch) ? + -ESTALE : 0; + if (status) + goto unlock_route; + if (mk_reply_claim(instance, &reply)) + goto unlock_route; + status = mk_pci_config_access(instance, &control_work->request, + &value, &reply); + if (mk_reply_publish_route_locked(instance, &reply, status, value)) + pr_warn_ratelimited("Failed to return PCI config response to instance %d\n", + instance->id); +unlock_route: + up_read(&instance->control_route_sem); + mk_instance_put(instance); +out: + mempool_free(control_work, mk_pci_control_pool); +} + +static void mk_pci_control_msg_handler(u32 msg_type, u32 subtype, + void *payload, u32 payload_len, + s32 sender_instance_id, void *ctx) +{ + struct mk_pci_control_work *work; + + if (msg_type != MK_MSG_PCI || subtype != MK_PCI_CFG_REQUEST || + payload_len != sizeof(work->request)) + return; + if (!mk_pci_control_handler_get()) + return; + + /* One reserve object exists for every valid outstanding reply slot. */ + work = mempool_alloc(mk_pci_control_pool, GFP_ATOMIC); + if (!work) { + atomic64_inc(&mk_pci_control_pool_exhausted); + pr_warn_ratelimited("Multikernel PCI control work pool exhausted\n"); + goto out; + } + INIT_WORK(&work->work, mk_pci_cfg_work_fn); + memcpy(&work->request, payload, sizeof(work->request)); + work->sender_instance_id = sender_instance_id; + /* Payload identity is diagnostic only; endpoint provenance is authority. */ + work->request.sender_instance_id = sender_instance_id; + if (!queue_work(mk_pci_control_wq, &work->work)) + mempool_free(work, mk_pci_control_pool); +out: + mk_pci_control_handler_put(); +} + static void mk_pci_assignment_failure_work(struct work_struct *work) { struct mk_pci_assignment *assignment = @@ -847,7 +1064,7 @@ static int mk_pci_commit_assignment(struct mk_pci_assignment *assignment) list_move_tail(&assignment->inventory->list, &assignment->instance->pci_devices); - root_instance->pci_device_count--; + mk_self->pci_device_count--; assignment->instance->pci_device_count++; assignment->instance->pci_devices_valid = true; assignment->inventory_moved = true; @@ -891,12 +1108,12 @@ static int mk_pci_release_assignment(struct mk_pci_assignment *assignment) mk_pci_iommu_release_assignment(assignment); cancel_work_sync(&assignment->failure_work); - if (assignment->inventory_moved && root_instance) { + if (assignment->inventory_moved && mk_self) { list_move_tail(&assignment->inventory->list, - &root_instance->pci_devices); + &mk_self->pci_devices); instance->pci_device_count--; - root_instance->pci_device_count++; - root_instance->pci_devices_valid = true; + mk_self->pci_device_count++; + mk_self->pci_devices_valid = true; assignment->inventory_moved = false; } @@ -980,10 +1197,10 @@ int mk_pci_assign_devices(struct mk_instance *instance, int ret = 0; int rollback_ret; - if (!instance || instance == root_instance || !requested_devices || + if (!instance || instance == mk_self || !requested_devices || requested_count < 0) return -EINVAL; - if (!root_instance || !root_instance->pci_devices_valid) + if (!mk_self || !mk_self->pci_devices_valid) return -EINVAL; mutex_lock(&instance->resource_mutex); @@ -1027,7 +1244,7 @@ int mk_pci_assign_device(struct mk_instance *instance, u16 domain, u8 bus, int ret; int rollback_ret; - if (!instance || instance == root_instance) + if (!instance || instance == mk_self) return -EINVAL; mutex_lock(&instance->resource_mutex); @@ -1038,7 +1255,7 @@ int mk_pci_assign_device(struct mk_instance *instance, u16 domain, u8 bus, ret = -EBUSY; goto out; } - if (!root_instance || !root_instance->pci_devices_valid) { + if (!mk_self || !mk_self->pci_devices_valid) { ret = -EINVAL; goto out; } @@ -1074,7 +1291,7 @@ int mk_pci_unassign_device(struct mk_instance *instance, u16 domain, u8 bus, struct mk_pci_assignment *assignment; int ret; - if (!instance || instance == root_instance) + if (!instance || instance == mk_self) return -EINVAL; mutex_lock(&instance->resource_mutex); @@ -1105,7 +1322,7 @@ int mk_pci_release_assignments(struct mk_instance *instance) struct mk_pci_assignment *assignment; int ret = 0; - if (!instance || instance == root_instance) + if (!instance || instance == mk_self) return 0; mutex_lock(&instance->resource_mutex); @@ -1133,7 +1350,7 @@ int mk_pci_prepare_instance_start(struct mk_instance *instance) struct mk_pci_assignment *assignment; int ret = 0; - if (!instance || instance == root_instance) + if (!instance || instance == mk_self) return -EINVAL; mutex_lock(&instance->resource_mutex); @@ -1153,6 +1370,8 @@ int mk_pci_prepare_instance_start(struct mk_instance *instance) int mk_pci_lease_system_init(void) { + unsigned int pool_size; + size_t work_size = sizeof(struct mk_pci_control_work); int ret; ret = mk_pci_iommu_system_init(); @@ -1164,11 +1383,59 @@ int mk_pci_lease_system_init(void) return ret; } mk_pci_notifier_registered = true; + if (mk_self && mk_self->id == 0) { + pool_size = max_t(unsigned int, num_possible_cpus(), 1) * + MK_REPLY_SLOTS; + mk_pci_control_pool = mempool_create_kmalloc_pool(pool_size, work_size); + if (!mk_pci_control_pool) { + ret = -ENOMEM; + goto unregister_notifier; + } + mk_pci_control_wq = + alloc_workqueue("mk-pci-control", + WQ_UNBOUND | WQ_MEM_RECLAIM, 0); + if (!mk_pci_control_wq) { + ret = -ENOMEM; + goto destroy_pool; + } + mk_pci_control_shutdown_end(); + ret = mk_register_msg_handler(MK_MSG_PCI, + mk_pci_control_msg_handler, NULL); + if (ret) + goto destroy_workqueue; + mk_pci_control_registered = true; + } return 0; + +destroy_workqueue: + mk_pci_control_shutdown_begin(); + destroy_workqueue(mk_pci_control_wq); + mk_pci_control_wq = NULL; +destroy_pool: + mempool_destroy(mk_pci_control_pool); + mk_pci_control_pool = NULL; +unregister_notifier: + bus_unregister_notifier(&pci_bus_type, &mk_pci_bus_notifier); + mk_pci_notifier_registered = false; + mk_pci_iommu_system_cleanup(); + return ret; } void mk_pci_lease_system_cleanup(void) { + if (mk_pci_control_registered) { + mk_pci_control_shutdown_begin(); + mk_unregister_msg_handler(MK_MSG_PCI, + mk_pci_control_msg_handler); + mk_pci_control_registered = false; + wait_event(mk_pci_control_waitq, !READ_ONCE(mk_pci_control_active)); + } + if (mk_pci_control_wq) { + destroy_workqueue(mk_pci_control_wq); + mk_pci_control_wq = NULL; + } + mempool_destroy(mk_pci_control_pool); + mk_pci_control_pool = NULL; if (mk_pci_notifier_registered) { bus_unregister_notifier(&pci_bus_type, &mk_pci_bus_notifier); mk_pci_notifier_registered = false; @@ -1179,11 +1446,10 @@ void mk_pci_lease_system_cleanup(void) static struct mk_pci_device * mk_pci_find_assigned_bdf(u16 domain, u8 bus, u8 devfn) { - if (!root_instance || !root_instance->dtb_data || - !root_instance->pci_devices_valid) + if (!mk_self || !mk_self->pci_devices_valid) return NULL; - return mk_pci_find_device_bdf(&root_instance->pci_devices, + return mk_pci_find_device_bdf(&mk_self->pci_devices, domain, bus, devfn); } From f5cde370ae61864a794dd6e678b5666585ac784f Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Tue, 11 Aug 2026 09:39:15 +0300 Subject: [PATCH 14/24] multikernel: move assigned MSI control to process context Keep MSI and MSI-X programming out of irqchip callbacks. Setup, bind, activation, restore, and teardown use the process-context PCI control path; write_msg only updates the cached message. Route controlled VF FLR through a separate process-context direct reply, with an independent spawn epoch and serial generation. Reject raw config-space FLR writes so reset cannot run inside the bounded atomic config path. Reject stale operations without side effects and fail closed on incomplete activation or reset. Pre-mask controlled MSI-X tables before host activation. Bump the exact transport ABI to version 6. Signed-off-by: Nikolay Nikolaev --- Documentation/multikernel/usage.rst | 2 + arch/x86/multikernel/pci.c | 459 ++++++++++++++++- drivers/pci/msi/api.c | 9 + drivers/pci/msi/irqdomain.c | 27 +- drivers/pci/msi/msi.c | 22 +- drivers/pci/pci.c | 3 + include/linux/multikernel.h | 90 ++++ include/linux/pci.h | 9 + kernel/multikernel/internal.h | 7 +- kernel/multikernel/pci.c | 752 +++++++++++++++++++++++++++- 10 files changed, 1339 insertions(+), 41 deletions(-) diff --git a/Documentation/multikernel/usage.rst b/Documentation/multikernel/usage.rst index ea153740658173..234c5a069d2eec 100644 --- a/Documentation/multikernel/usage.rst +++ b/Documentation/multikernel/usage.rst @@ -97,3 +97,5 @@ Restrictions - Instance files are read-only; an instance's resources change through overlays targeting ``/instances/``. - Rollback (``rmdir`` on a transaction) cannot destroy a running instance. +- Logical CPU 0 handles assigned-device MSI forwarding and must remain online + while those devices are active. diff --git a/arch/x86/multikernel/pci.c b/arch/x86/multikernel/pci.c index 7ebbfa45e762fa..7a5b5049b11b12 100644 --- a/arch/x86/multikernel/pci.c +++ b/arch/x86/multikernel/pci.c @@ -2,11 +2,16 @@ /* * x86 PCI support for multikernel spawn kernels. * - * Spawn kernels discover only assigned BDFs. Configuration accesses are - * filtered here and become host-mediated once the RPC transport is installed. + * Spawn kernels discover only assigned BDFs and proxy all configuration + * accesses to the host kernel. */ + #include +#include +#include +#include #include +#include #include #include #include @@ -20,6 +25,425 @@ static atomic64_t mk_pci_request_id = ATOMIC64_INIT(0); static atomic64_t mk_pci_cfg_count = ATOMIC64_INIT(0); static atomic64_t mk_pci_cfg_total_ns = ATOMIC64_INIT(0); static atomic64_t mk_pci_cfg_max_ns = ATOMIC64_INIT(0); +#define MK_PCI_RESET_TIMEOUT_MS 70000 +#ifdef CONFIG_PCI_MSI + +static bool mk_pci_message_from_host(s32 sender_instance_id) +{ + return host_instance && sender_instance_id == host_instance->id; +} + +static void mk_pci_forward_irq_noop(struct irq_data *data) +{ +} + +static void mk_pci_forward_irq_write_msg(struct irq_data *data, + struct msi_msg *msg) +{ +} + +static struct irq_chip mk_pci_forward_irq_chip = { + .name = "multikernel-pci-forward", + .irq_ack = mk_pci_forward_irq_noop, + /* Host process-context lifecycle owns physical mask state. */ + .irq_mask = mk_pci_forward_irq_noop, + .irq_unmask = mk_pci_forward_irq_noop, + .irq_write_msi_msg = mk_pci_forward_irq_write_msg, +}; + +static void mk_pci_bind_local_irqs(unsigned int irq, unsigned int count) +{ + unsigned int i; + + for (i = 0; i < count; i++) + irq_set_chip_and_handler(irq + i, &mk_pci_forward_irq_chip, + handle_edge_irq); +} + +static bool mk_pci_forward_irq_matches(const struct mk_io_irq_payload *irq, + struct irq_data **irq_data) +{ + struct irq_data *data = irq_get_irq_data(irq->irq_number); + struct msi_desc *desc; + struct pci_dev *dev; + unsigned int offset; + + if (!data) + return false; + desc = irq_data_get_msi_desc(data); + if (!desc || irq->vector < desc->msi_index) + return false; + + dev = msi_desc_to_pci_dev(desc); + offset = irq->vector - desc->msi_index; + if (offset >= desc->nvec_used || desc->irq + offset != irq->irq_number || + pci_domain_nr(dev->bus) != MK_PCI_IRQ_ID_DOMAIN(irq->device_id) || + dev->bus->number != MK_PCI_IRQ_ID_BUS(irq->device_id) || + dev->devfn != MK_PCI_IRQ_ID_DEVFN(irq->device_id)) + return false; + + *irq_data = data; + return true; +} + +static void mk_pci_irq_forward_handler(u32 msg_type, u32 subtype, + void *payload, u32 payload_len, + s32 sender_instance_id, void *ctx) +{ + struct mk_io_irq_payload *irq = payload; + struct irq_data *irq_data; + struct pci_dev *dev; + + if (msg_type != MK_MSG_IO || subtype != MK_IO_IRQ_FORWARD || + payload_len != sizeof(*irq) || + !mk_pci_message_from_host(sender_instance_id)) + return; + if (!mk_pci_forward_irq_matches(irq, &irq_data)) { + pr_warn_ratelimited("Rejected host-forwarded PCI IRQ %u with unmatched identity %#x vector %u\n", + irq->irq_number, irq->device_id, + irq->vector); + return; + } + dev = msi_desc_to_pci_dev(irq_data_get_msi_desc(irq_data)); + if (READ_ONCE(dev->multikernel_msi_state) != MK_PCI_MSI_ACTIVE || + irq->lifecycle_generation != + READ_ONCE(dev->multikernel_msi_generation) || + !mk_self->ipi_data || + irq->lifecycle_epoch != + READ_ONCE(mk_self->ipi_data->spawn_epoch)) + return; + + if (irq_data_get_irq_chip(irq_data) != &mk_pci_forward_irq_chip) { + pr_warn_ratelimited("Rejected host-forwarded PCI IRQ %u for %s vector %u before local binding\n", + irq->irq_number, pci_name(dev), + irq->vector); + return; + } + + if (generic_handle_irq_safe(irq->irq_number)) + pr_warn_ratelimited("Failed to dispatch host-forwarded PCI IRQ %u\n", + irq->irq_number); +} + +static int mk_pci_send_irq_request(struct mk_pci_irq_request *request) +{ + struct mk_reply_handle reply; + s32 status; + int ret; + + if (WARN_ON_ONCE(irqs_disabled() || !in_task())) + return -EWOULDBLOCK; + might_sleep(); + if (!request->lifecycle_generation || !mk_self || + !mk_self->ipi_data) + return -EINVAL; + request->lifecycle_epoch = + READ_ONCE(mk_self->ipi_data->spawn_epoch); + if (!request->lifecycle_epoch) + return -EPROTO; + + request->request_id = atomic64_inc_return(&mk_pci_request_id); + request->sender_instance_id = mk_self ? mk_self->id : -1; + ret = mk_reply_reserve(mk_self->ipi_data, MK_REPLY_PCI_IRQ, + request->request_id, &reply); + if (ret) + return ret; + request->reply_slot = reply.slot; + request->reply_generation = reply.generation; + + ret = mk_send_message(host_instance->id, MK_MSG_PCI, MK_PCI_IRQ_REQUEST, + request, sizeof(*request)); + if (ret) { + mk_reply_release(mk_self->ipi_data, &reply); + return ret; + } + + ret = mk_reply_wait(mk_self->ipi_data, &reply, 1000, + &status, NULL); + return ret ? ret : status; +} + +bool mk_pci_msi_controlled(struct pci_dev *dev) +{ + return mk_pci_controlled(dev); +} + +static int mk_pci_msi_teardown_generation(struct pci_dev *dev, + u32 generation); + +int mk_pci_msi_prepare(struct pci_dev *dev, int nvec, int type) +{ + u32 generation; + u8 state; + struct mk_pci_irq_request request = { + .domain = pci_domain_nr(dev->bus), + .bus = dev->bus->number, + .devfn = dev->devfn, + .operation = MK_PCI_IRQ_SETUP, + .nr_vectors = nvec, + .msix = type == PCI_CAP_ID_MSIX, + }; + int ret; + + if (!mk_pci_msi_controlled(dev)) + return 0; + state = READ_ONCE(dev->multikernel_msi_state); + if (state != MK_PCI_MSI_IDLE && state != MK_PCI_MSI_FAILED) + return -EBUSY; + generation = READ_ONCE(dev->multikernel_msi_generation) + 1; + if (!generation) + generation = 1; + WRITE_ONCE(dev->multikernel_msi_generation, generation); + request.lifecycle_generation = generation; + ret = mk_pci_send_irq_request(&request); + if (ret) { + if (ret == -EINPROGRESS) { + int cleanup_ret; + + cleanup_ret = mk_pci_msi_teardown_generation(dev, generation); + WRITE_ONCE(dev->multikernel_msi_state, + cleanup_ret ? MK_PCI_MSI_FAILED : + MK_PCI_MSI_IDLE); + } + return ret; + } + WRITE_ONCE(dev->multikernel_msi_nvec, nvec); + WRITE_ONCE(dev->multikernel_msi_msix, type == PCI_CAP_ID_MSIX); + WRITE_ONCE(dev->multikernel_msi_state, MK_PCI_MSI_PREPARED); + return 0; +} + +static int mk_pci_msi_bind(struct pci_dev *dev, unsigned int index, + unsigned int irq, unsigned int nvec, bool msix, + u32 generation) +{ + struct mk_pci_irq_request request = { + .domain = pci_domain_nr(dev->bus), + .bus = dev->bus->number, + .devfn = dev->devfn, + .operation = MK_PCI_IRQ_BIND, + .vector = index, + .nr_vectors = nvec, + .msix = msix, + .local_irq = irq, + .lifecycle_generation = generation, + }; + + /* The local descriptor must be dispatchable before the host unmasks. */ + mk_pci_bind_local_irqs(irq, msix ? 1 : nvec); + return mk_pci_send_irq_request(&request); +} + +static int mk_pci_msi_teardown_generation(struct pci_dev *dev, u32 generation) +{ + struct mk_pci_irq_request request = { + .domain = pci_domain_nr(dev->bus), + .bus = dev->bus->number, + .devfn = dev->devfn, + .operation = MK_PCI_IRQ_TEARDOWN, + .lifecycle_generation = generation, + }; + + return mk_pci_send_irq_request(&request); +} + +static int mk_pci_msi_commit(struct pci_dev *dev, u32 generation) +{ + struct mk_pci_irq_request request = { + .domain = pci_domain_nr(dev->bus), + .bus = dev->bus->number, + .devfn = dev->devfn, + .operation = MK_PCI_IRQ_COMMIT, + .lifecycle_generation = generation, + }; + + return mk_pci_send_irq_request(&request); +} + +static int mk_pci_msi_host_activate(struct pci_dev *dev, u32 generation) +{ + struct mk_pci_irq_request request = { + .domain = pci_domain_nr(dev->bus), + .bus = dev->bus->number, + .devfn = dev->devfn, + .operation = MK_PCI_IRQ_ACTIVATE, + .lifecycle_generation = generation, + }; + + return mk_pci_send_irq_request(&request); +} + +static int mk_pci_msi_restore_begin(struct pci_dev *dev, u32 generation) +{ + struct mk_pci_irq_request request = { + .domain = pci_domain_nr(dev->bus), + .bus = dev->bus->number, + .devfn = dev->devfn, + .operation = MK_PCI_IRQ_RESTORE_BEGIN, + .lifecycle_generation = generation, + }; + + return mk_pci_send_irq_request(&request); +} + +static int mk_pci_msi_bind_all(struct pci_dev *dev, u32 generation) +{ + struct msi_desc *desc; + unsigned int expected = READ_ONCE(dev->multikernel_msi_nvec); + unsigned int next = 0; + bool msix = READ_ONCE(dev->multikernel_msi_msix); + int ret; + + msi_for_each_desc(desc, &dev->dev, MSI_DESC_ALL) { + unsigned int count = msix ? 1 : desc->nvec_used; + + if (next >= expected || desc->msi_index != next || !count || + count > expected - next) + return -EINVAL; + ret = mk_pci_msi_bind(dev, desc->msi_index, desc->irq, + desc->nvec_used, msix, generation); + if (ret) + return ret; + next += count; + } + if (next != expected) + return -EINVAL; + return mk_pci_msi_commit(dev, generation); +} + +int mk_pci_msi_activate(struct pci_dev *dev) +{ + u32 generation; + int cleanup_ret; + int ret; + + if (!mk_pci_msi_controlled(dev)) + return 0; + if (READ_ONCE(dev->multikernel_msi_state) != MK_PCI_MSI_PREPARED) + return -EIO; + generation = READ_ONCE(dev->multikernel_msi_generation); + ret = mk_pci_msi_bind_all(dev, generation); + if (ret) { + pr_err("Failed to activate host-owned MSI vectors for %s: %d\n", + pci_name(dev), ret); + cleanup_ret = mk_pci_msi_teardown_generation(dev, generation); + WRITE_ONCE(dev->multikernel_msi_state, + cleanup_ret ? MK_PCI_MSI_FAILED : MK_PCI_MSI_IDLE); + return ret; + } + + /* The guest must be able to consume the first edge before host unmask. */ + WRITE_ONCE(dev->multikernel_msi_state, MK_PCI_MSI_ACTIVE); + ret = mk_pci_msi_host_activate(dev, generation); + if (ret) { + cleanup_ret = mk_pci_msi_teardown_generation(dev, generation); + WRITE_ONCE(dev->multikernel_msi_state, + cleanup_ret ? MK_PCI_MSI_FAILED : MK_PCI_MSI_IDLE); + return ret; + } + return 0; +} + +int mk_pci_msi_restore(struct pci_dev *dev) +{ + u32 generation; + int cleanup_ret; + int ret; + + if (!mk_pci_msi_controlled(dev)) + return 0; + if (READ_ONCE(dev->multikernel_msi_state) != MK_PCI_MSI_ACTIVE) + return -EIO; + generation = READ_ONCE(dev->multikernel_msi_generation); + ret = mk_pci_msi_restore_begin(dev, generation); + if (!ret) + ret = mk_pci_msi_bind_all(dev, generation); + if (!ret) + ret = mk_pci_msi_host_activate(dev, generation); + if (ret) { + cleanup_ret = mk_pci_msi_teardown_generation(dev, generation); + WRITE_ONCE(dev->multikernel_msi_state, MK_PCI_MSI_FAILED); + if (cleanup_ret) + pr_err("Failed to quiesce host-owned MSI after restore failure for %s: %d\n", + pci_name(dev), cleanup_ret); + return ret; + } + return 0; +} + +int mk_pci_msi_teardown(struct pci_dev *dev) +{ + u32 generation; + int ret; + + if (!mk_pci_msi_controlled(dev)) + return 0; + if (READ_ONCE(dev->multikernel_msi_state) == MK_PCI_MSI_IDLE) + return 0; + generation = READ_ONCE(dev->multikernel_msi_generation); + ret = mk_pci_msi_teardown_generation(dev, generation); + WRITE_ONCE(dev->multikernel_msi_state, + ret ? MK_PCI_MSI_FAILED : MK_PCI_MSI_IDLE); + return ret; +} +#endif /* CONFIG_PCI_MSI */ + +bool mk_pci_controlled(struct pci_dev *dev) +{ + return mk_self && mk_self->id != 0 && + mk_pci_get_assigned_identity_bdf(pci_domain_nr(dev->bus), + dev->bus->number, dev->devfn, + NULL, NULL); +} + +int mk_pci_reset_flr(struct pci_dev *dev) +{ + struct mk_pci_reset_request request = { + .domain = pci_domain_nr(dev->bus), + .bus = dev->bus->number, + .devfn = dev->devfn, + }; + struct mk_reply_handle reply; + u32 generation; + s32 status; + int ret; + + if (WARN_ON_ONCE(irqs_disabled() || !in_task())) + return -EWOULDBLOCK; + might_sleep(); + if (!mk_self || !mk_self->ipi_data) + return -ENODEV; + request.lifecycle_epoch = + READ_ONCE(mk_self->ipi_data->spawn_epoch); + if (!request.lifecycle_epoch) + return -EPROTO; + + generation = READ_ONCE(dev->multikernel_reset_generation) + 1; + if (!generation) + generation = 1; + WRITE_ONCE(dev->multikernel_reset_generation, generation); + request.reset_generation = generation; + request.request_id = atomic64_inc_return(&mk_pci_request_id); + request.sender_instance_id = mk_self->id; + ret = mk_reply_reserve(mk_self->ipi_data, MK_REPLY_PCI_RESET, + request.request_id, &reply); + if (ret) + return ret; + request.reply_slot = reply.slot; + request.reply_generation = reply.generation; + + ret = mk_send_message(host_instance->id, MK_MSG_PCI, MK_PCI_RESET_REQUEST, + &request, sizeof(request)); + if (ret) { + mk_reply_release(mk_self->ipi_data, &reply); + return ret; + } + + ret = mk_reply_wait(mk_self->ipi_data, &reply, + MK_PCI_RESET_TIMEOUT_MS, + &status, NULL); + return ret ? ret : status; +} static void mk_pci_record_latency(u64 start) { @@ -71,6 +495,7 @@ static int mk_pci_remote_config(unsigned int domain, unsigned int bus, goto out_error; request.reply_slot = reply.slot; request.reply_generation = reply.generation; + ret = mk_send_message(host_instance->id, MK_MSG_PCI, MK_PCI_CFG_REQUEST, &request, sizeof(request)); if (ret) { @@ -94,7 +519,6 @@ static int mk_pci_remote_config(unsigned int domain, unsigned int bus, if (ret < 0) { pr_err_ratelimited("Multikernel PCI config request timed out or failed to send: %d\n", ret); - return PCIBIOS_SET_FAILED; } return PCIBIOS_SET_FAILED; } @@ -107,6 +531,7 @@ static bool mk_pci_identity_read(u16 vendor, u16 device, int where, int size, if (where < PCI_VENDOR_ID || where + size > PCI_COMMAND) return false; + identity = vendor | (u32)device << 16; mask = size == sizeof(identity) ? ~0U : (1U << (size * 8)) - 1; *value = (identity >> (where * 8)) & mask; @@ -126,6 +551,7 @@ static int mk_pci_raw_read(unsigned int domain, unsigned int bus, } if (mk_pci_identity_read(vendor, device, where, size, value)) return PCIBIOS_SUCCESSFUL; + return mk_pci_remote_config(domain, bus, devfn, where, size, false, value); } @@ -136,6 +562,7 @@ static int mk_pci_raw_write(unsigned int domain, unsigned int bus, { if (!mk_pci_get_assigned_identity_bdf(domain, bus, devfn, NULL, NULL)) return PCIBIOS_DEVICE_NOT_FOUND; + return mk_pci_remote_config(domain, bus, devfn, where, size, true, &value); } @@ -144,14 +571,24 @@ static const struct pci_raw_ops mk_pci_filtered_raw_ops = { .read = mk_pci_raw_read, .write = mk_pci_raw_write, }; + static int __init x86_multikernel_pci_arch_init(void) { if (!mk_self || !host_instance || !mk_self->pci_devices_valid) return 0; +#ifdef CONFIG_PCI_MSI + if (mk_register_msg_handler(MK_MSG_IO, mk_pci_irq_forward_handler, + NULL)) { + pr_err("Multikernel failed to register PCI IRQ forwarding handler\n"); + return 0; + } +#endif + raw_pci_ops = &mk_pci_filtered_raw_ops; raw_pci_ext_ops = &mk_pci_filtered_raw_ops; mk_pci_roots_ready = true; - pr_notice("Multikernel selected filtered raw PCI config access\n"); + pr_notice("Multikernel selected host-mediated PCI config access\n"); + return 0; } @@ -191,11 +628,12 @@ static struct pci_bus * __init mk_pci_get_root(u16 domain, u8 bus_number) kfree(sd); return ERR_PTR(-ENOMEM); } - if (!has_busn_res && - !pci_bus_insert_busn_res(bus, bus_number, bus_number)) { - pci_remove_root_bus(bus); - kfree(sd); - return ERR_PTR(-EBUSY); + if (!has_busn_res) { + if (!pci_bus_insert_busn_res(bus, bus_number, bus_number)) { + pci_remove_root_bus(bus); + kfree(sd); + return ERR_PTR(-EBUSY); + } } pr_notice("Multikernel created synthetic PCI root %04x:%02x\n", domain, bus_number); @@ -236,10 +674,9 @@ static int __init x86_multikernel_pci_init(void) atomic64_read(&mk_pci_cfg_max_ns)); } - /* Suppress legacy bus 0 probing after every assigned function is present. */ + /* Suppress legacy bus 0 probing after every assigned root is present. */ return 0; } - void __init x86_multikernel_pci_platform_init(void) { pci_probe = PCI_PROBE_NOEARLY; diff --git a/drivers/pci/msi/api.c b/drivers/pci/msi/api.c index 818d55fbad0dfa..ca7195c58ef8e0 100644 --- a/drivers/pci/msi/api.c +++ b/drivers/pci/msi/api.c @@ -10,6 +10,7 @@ #include #include +#include #include "msi.h" @@ -123,6 +124,8 @@ bool pci_msix_can_alloc_dyn(struct pci_dev *dev) { if (!dev->msix_cap) return false; + if (mk_pci_msi_controlled(dev)) + return false; return pci_msi_domain_supports(dev, MSI_FLAG_PCI_MSIX_ALLOC_DYN, DENY_LEGACY); } @@ -388,6 +391,12 @@ EXPORT_SYMBOL(pci_free_irq_vectors); */ void pci_restore_msi_state(struct pci_dev *dev) { + if (mk_pci_msi_controlled(dev)) { + if (mk_pci_msi_restore(dev)) + pr_err("Failed to restore host-owned MSI vectors for %s\n", + pci_name(dev)); + return; + } __pci_restore_msi_state(dev); __pci_restore_msix_state(dev); } diff --git a/drivers/pci/msi/irqdomain.c b/drivers/pci/msi/irqdomain.c index 6e65f0f44112e6..e79bf05f07a325 100644 --- a/drivers/pci/msi/irqdomain.c +++ b/drivers/pci/msi/irqdomain.c @@ -4,6 +4,7 @@ */ #include #include +#include #include #include "msi.h" @@ -11,18 +12,38 @@ int pci_msi_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) { struct irq_domain *domain; + bool controlled = mk_pci_msi_controlled(dev); + int ret; + + if (controlled) { + ret = mk_pci_msi_prepare(dev, nvec, type); + if (ret) + return ret; + } domain = dev_get_msi_domain(&dev->dev); if (domain && irq_domain_is_hierarchy(domain)) - return msi_domain_alloc_irqs_all_locked(&dev->dev, MSI_DEFAULT_DOMAIN, nvec); - - return pci_msi_legacy_setup_msi_irqs(dev, nvec, type); + ret = msi_domain_alloc_irqs_all_locked(&dev->dev, + MSI_DEFAULT_DOMAIN, nvec); + else + ret = pci_msi_legacy_setup_msi_irqs(dev, nvec, type); + if (!ret && controlled) + ret = mk_pci_msi_activate(dev); + + if (ret && controlled && mk_pci_msi_teardown(dev)) + pr_err("Failed to roll back host-owned MSI setup for %s\n", + pci_name(dev)); + return ret; } void pci_msi_teardown_msi_irqs(struct pci_dev *dev) { struct irq_domain *domain; + if (mk_pci_msi_controlled(dev) && mk_pci_msi_teardown(dev)) + pr_err("Failed to tear down host-owned MSI vectors for %s\n", + pci_name(dev)); + domain = dev_get_msi_domain(&dev->dev); if (domain && irq_domain_is_hierarchy(domain)) { msi_domain_free_irqs_all_locked(&dev->dev, MSI_DEFAULT_DOMAIN); diff --git a/drivers/pci/msi/msi.c b/drivers/pci/msi/msi.c index e2412175d7af20..6694a75e8ac92b 100644 --- a/drivers/pci/msi/msi.c +++ b/drivers/pci/msi/msi.c @@ -11,6 +11,7 @@ #include #include #include +#include #include "../pci.h" #include "msi.h" @@ -240,6 +241,14 @@ void __pci_write_msi_msg(struct msi_desc *entry, struct msi_msg *msg) { struct pci_dev *dev = msi_desc_to_pci_dev(entry); + if (mk_pci_msi_controlled(dev)) { + /* Host programming is committed later from the setup context. */ + entry->msg = *msg; + if (entry->write_msi_msg) + entry->write_msi_msg(entry, entry->write_msi_msg_data); + return; + } + if (dev->current_state != PCI_D0 || pci_dev_is_disconnected(dev)) { /* Don't touch the hardware now */ } else if (entry->pci.msi_attrib.is_msix) { @@ -714,6 +723,7 @@ static int msix_setup_interrupts(struct pci_dev *dev, struct msix_entry *entries static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries, int nvec, struct irq_affinity *affd) { + bool controlled = mk_pci_msi_controlled(dev); int ret, tsize; u16 control; @@ -736,6 +746,15 @@ static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries, ret = -ENOMEM; goto out_disable; } + if (controlled && + !pci_msi_domain_supports(dev, MSI_FLAG_NO_MASK, DENY_LEGACY)) { + /* + * The controlled setup path activates host-owned vectors from + * msix_setup_interrupts(). Clear stale table entries while the + * function-wide mask is still set, before host activation. + */ + msix_mask_all(dev->msix_base, tsize); + } ret = msix_setup_interrupts(dev, entries, nvec, affd); if (ret) @@ -744,7 +763,8 @@ static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries, /* Disable INTX */ pci_intx_for_msi(dev, 0); - if (!pci_msi_domain_supports(dev, MSI_FLAG_NO_MASK, DENY_LEGACY)) { + if (!controlled && + !pci_msi_domain_supports(dev, MSI_FLAG_NO_MASK, DENY_LEGACY)) { /* * Ensure that all table entries are masked to prevent * stale entries from firing in a crash kernel. diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index 8479c2e1f74f10..7a9f7a327971ef 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -4380,6 +4381,8 @@ int pcie_reset_flr(struct pci_dev *dev, bool probe) if (probe) return 0; + if (mk_pci_controlled(dev)) + return mk_pci_reset_flr(dev); return pcie_flr(dev); } diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index a8413239d3a163..f95612a38e6050 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -101,6 +101,7 @@ enum mk_reply_state { enum mk_reply_kind { MK_REPLY_PCI_CFG = 1, MK_REPLY_PCI_IRQ, + MK_REPLY_PCI_RESET, }; struct mk_reply_slot { @@ -352,6 +353,10 @@ void mk_reply_scan(struct mk_shared_data *shared); /* Host-mediated PCI control-plane subtypes */ #define MK_PCI_CFG_REQUEST (MK_MSG_PCI + 1) #define MK_PCI_CFG_RESPONSE (MK_MSG_PCI + 2) +#define MK_PCI_IRQ_REQUEST (MK_MSG_PCI + 3) +#define MK_PCI_IRQ_RESPONSE (MK_MSG_PCI + 4) +#define MK_PCI_RESET_REQUEST (MK_MSG_PCI + 5) +#define MK_PCI_RESET_RESPONSE (MK_MSG_PCI + 6) /** * Core message structure @@ -374,8 +379,17 @@ struct mk_io_irq_payload { u32 vector; /* Interrupt vector */ u32 device_id; /* Device identifier (optional) */ u32 flags; /* Control flags (priority, etc.) */ + u32 lifecycle_generation; + u32 reserved; + u64 lifecycle_epoch; }; +#define MK_PCI_IRQ_ID(domain, bus, devfn) \ + (((u32)(domain) << 16) | ((u32)(bus) << 8) | (u32)(devfn)) +#define MK_PCI_IRQ_ID_DOMAIN(id) ((u16)((id) >> 16)) +#define MK_PCI_IRQ_ID_BUS(id) ((u8)((id) >> 8)) +#define MK_PCI_IRQ_ID_DEVFN(id) ((u8)(id)) + struct mk_pci_cfg_request { u64 request_id; u64 lifecycle_epoch; @@ -398,6 +412,54 @@ struct mk_pci_cfg_response { u32 value; }; +enum mk_pci_irq_operation { + MK_PCI_IRQ_SETUP = 1, + MK_PCI_IRQ_RESTORE_BEGIN, + MK_PCI_IRQ_BIND, + MK_PCI_IRQ_COMMIT, + MK_PCI_IRQ_ACTIVATE, + MK_PCI_IRQ_TEARDOWN, +}; + +enum mk_pci_msi_lifecycle { + MK_PCI_MSI_IDLE = 0, + MK_PCI_MSI_PREPARED, + MK_PCI_MSI_COMMITTED, + MK_PCI_MSI_ACTIVE, + MK_PCI_MSI_FAILED, +}; + +struct mk_pci_irq_request { + u64 request_id; + s32 sender_instance_id; + u16 domain; + u8 bus; + u8 devfn; + u16 operation; + u16 vector; + u16 nr_vectors; + u8 msix; + u8 reserved; + u32 local_irq; + u32 reply_slot; + u32 lifecycle_generation; + u64 reply_generation; + u64 lifecycle_epoch; +}; + +struct mk_pci_reset_request { + u64 request_id; + s32 sender_instance_id; + u16 domain; + u8 bus; + u8 devfn; + u32 reset_generation; + u32 reply_slot; + u32 reserved; + u64 reply_generation; + u64 lifecycle_epoch; +}; + /* IRQ control flags */ #define MK_IRQ_HIGH_PRIORITY 0x01 #define MK_IRQ_LOW_LATENCY 0x02 @@ -1127,6 +1189,26 @@ bool mk_pci_should_probe(struct pci_bus *bus, int devfn); bool mk_pci_get_assigned_identity_bdf(unsigned int domain, unsigned int bus, unsigned int devfn, u16 *vendor, u16 *device); +#if defined(CONFIG_X86) && defined(CONFIG_PCI) +bool mk_pci_controlled(struct pci_dev *dev); +int mk_pci_reset_flr(struct pci_dev *dev); +#else +static inline bool mk_pci_controlled(struct pci_dev *dev) { return false; } +static inline int mk_pci_reset_flr(struct pci_dev *dev) { return -EOPNOTSUPP; } +#endif +#if defined(CONFIG_X86) && defined(CONFIG_PCI) && defined(CONFIG_PCI_MSI) +bool mk_pci_msi_controlled(struct pci_dev *dev); +int mk_pci_msi_prepare(struct pci_dev *dev, int nvec, int type); +int mk_pci_msi_activate(struct pci_dev *dev); +int mk_pci_msi_restore(struct pci_dev *dev); +int mk_pci_msi_teardown(struct pci_dev *dev); +#else +static inline bool mk_pci_msi_controlled(struct pci_dev *dev) { return false; } +static inline int mk_pci_msi_prepare(struct pci_dev *dev, int nvec, int type) { return 0; } +static inline int mk_pci_msi_activate(struct pci_dev *dev) { return 0; } +static inline int mk_pci_msi_restore(struct pci_dev *dev) { return 0; } +static inline int mk_pci_msi_teardown(struct pci_dev *dev) { return 0; } +#endif bool mk_platform_device_allowed(const char *name, const char *hid); /* Early CPU registration from the manifest (spawn kernels) */ @@ -1153,6 +1235,14 @@ static inline bool mk_crash_notes_wanted(void) return false; } +static inline bool mk_pci_controlled(struct pci_dev *dev) { return false; } +static inline int mk_pci_reset_flr(struct pci_dev *dev) { return -EOPNOTSUPP; } +static inline bool mk_pci_msi_controlled(struct pci_dev *dev) { return false; } +static inline int mk_pci_msi_prepare(struct pci_dev *dev, int nvec, int type) { return 0; } +static inline int mk_pci_msi_activate(struct pci_dev *dev) { return 0; } +static inline int mk_pci_msi_restore(struct pci_dev *dev) { return 0; } +static inline int mk_pci_msi_teardown(struct pci_dev *dev) { return 0; } + static inline int multikernel_force_halt_by_id(int mk_id) { return -ENODEV; diff --git a/include/linux/pci.h b/include/linux/pci.h index 1c270f1d512301..591b35215bdd08 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -503,6 +503,9 @@ struct pci_dev { unsigned int non_mappable_bars:1; /* BARs can't be mapped to user-space */ pci_dev_flags_t dev_flags; atomic_t enable_cnt; /* pci_enable_device has been called */ +#ifdef CONFIG_MULTIKERNEL + u32 multikernel_reset_generation; +#endif spinlock_t pcie_cap_lock; /* Protects RMW ops in capability accessors */ u32 saved_config_space[16]; /* Config space saved at suspend time */ @@ -524,6 +527,12 @@ struct pci_dev { #ifdef CONFIG_PCI_MSI void __iomem *msix_base; raw_spinlock_t msi_lock; +#ifdef CONFIG_MULTIKERNEL + u32 multikernel_msi_generation; + u16 multikernel_msi_nvec; + u8 multikernel_msi_state; + u8 multikernel_msi_msix; +#endif #endif struct pci_vpd vpd; #ifdef CONFIG_PCIE_DPC diff --git a/kernel/multikernel/internal.h b/kernel/multikernel/internal.h index 20420941ce4694..f2199d8cd2c723 100644 --- a/kernel/multikernel/internal.h +++ b/kernel/multikernel/internal.h @@ -79,11 +79,8 @@ int mk_pci_assign_device(struct mk_instance *instance, u16 domain, u8 bus, int mk_pci_unassign_device(struct mk_instance *instance, u16 domain, u8 bus, u8 devfn); int mk_pci_release_assignments(struct mk_instance *instance); -static inline unsigned int -mk_pci_sync_instance_irq_route(struct mk_instance *instance) -{ - return 0; -} +void mk_pci_quiesce_instance_irqs(struct mk_instance *instance); +unsigned int mk_pci_sync_instance_irq_route(struct mk_instance *instance); /* overlay.c */ extern struct kernfs_node *mk_overlay_root_kn; extern struct mutex mk_overlay_mutex; diff --git a/kernel/multikernel/pci.c b/kernel/multikernel/pci.c index 13e3cbe1830768..8418057eb3e759 100644 --- a/kernel/multikernel/pci.c +++ b/kernel/multikernel/pci.c @@ -7,6 +7,8 @@ * the current instance. */ +#include +#include #include #include #include @@ -21,6 +23,19 @@ #include "internal.h" +struct mk_pci_assignment; + +#define MK_PCI_FLR_SETTLE_MS 100 + +struct mk_pci_irq_vector { + struct mk_pci_assignment *assignment; + unsigned int host_irq; + u32 local_irq; + atomic64_t forwarded; + bool requested; + bool disabled; +}; + struct mk_pci_assignment { struct list_head instance_node; struct list_head active_node; @@ -38,6 +53,17 @@ struct mk_pci_assignment { bool iommu_dma_owner; bool iommu_attached; bool iommu_override_active; + struct mk_pci_irq_vector *irq_vectors; + unsigned long *irq_bound_map; + unsigned int nr_irq_vectors; + u64 irq_epoch; + u32 irq_generation; + u32 reset_generation; + u8 irq_state; + bool irq_msix; + bool irq_needs_reprogram; + unsigned long irq_flr_deadline; + bool device_enabled; struct work_struct failure_work; atomic_t failure_pending; bool assigned; @@ -60,10 +86,16 @@ static atomic64_t mk_pci_control_pool_exhausted = ATOMIC64_INIT(0); struct mk_pci_control_work { struct work_struct work; - struct mk_pci_cfg_request request; s32 sender_instance_id; + union { + struct mk_pci_cfg_request cfg; + struct mk_pci_irq_request irq; + struct mk_pci_reset_request reset; + } request; }; +static void mk_pci_schedule_failure(struct mk_pci_assignment *assignment); + static bool mk_pci_control_handler_get(void) { unsigned long flags; @@ -119,6 +151,15 @@ static bool mk_pci_device_live(struct pci_dev *pdev) pci_device_is_present(pdev); } +static int mk_pci_forwarding_cpu(void) +{ + /* The host IPI manifest publishes logical CPU 0's physical ID. */ + if (!cpu_online(0)) + return -ENODEV; + + return 0; +} + static bool mk_pci_device_matches_bdf(const struct mk_pci_device *device, u16 domain, u8 bus, u8 devfn) { @@ -185,6 +226,408 @@ mk_pci_find_assignment(struct mk_instance *instance, u16 domain, u8 bus, return NULL; } +static irqreturn_t mk_pci_forward_irq(int irq, void *data) +{ + struct mk_pci_irq_vector *vector = data; + struct mk_pci_assignment *assignment = vector->assignment; + struct mk_io_irq_payload payload = { + .vector = vector - assignment->irq_vectors, + .device_id = MK_PCI_IRQ_ID(pci_domain_nr(assignment->vf->bus), + assignment->vf->bus->number, + assignment->vf->devfn), + .flags = MK_IRQ_LOW_LATENCY | MK_IRQ_EDGE_TRIGGERED, + .lifecycle_generation = READ_ONCE(assignment->irq_generation), + .lifecycle_epoch = READ_ONCE(assignment->irq_epoch), + }; + u32 local_irq = READ_ONCE(vector->local_irq); + + if (READ_ONCE(assignment->instance->state) != MK_STATE_ACTIVE || + READ_ONCE(assignment->irq_state) != MK_PCI_MSI_ACTIVE || + !local_irq) + return IRQ_HANDLED; + + payload.irq_number = local_irq; + if (atomic64_inc_return(&vector->forwarded) == 1) + pr_info("Forwarding host IRQ %u as instance IRQ %u for %s vector %u\n", + irq, local_irq, pci_name(assignment->vf), + payload.vector); + if (mk_send_message_to_instance(assignment->instance, MK_MSG_IO, + MK_IO_IRQ_FORWARD, &payload, + sizeof(payload))) + pr_warn_ratelimited("Failed to forward IRQ %u for %s to instance %d\n", + irq, pci_name(assignment->vf), + assignment->instance->id); + return IRQ_HANDLED; +} + +static unsigned int mk_pci_quiesce_irqs(struct mk_pci_assignment *assignment) +{ + unsigned int disabled = 0; + unsigned int i; + + for (i = 0; i < assignment->nr_irq_vectors; i++) + WRITE_ONCE(assignment->irq_vectors[i].local_irq, 0); + + for (i = 0; i < assignment->nr_irq_vectors; i++) { + struct mk_pci_irq_vector *vector = &assignment->irq_vectors[i]; + + if (vector->requested && !vector->disabled) { + disable_irq(vector->host_irq); + vector->disabled = true; + disabled++; + } + } + if (assignment->irq_bound_map) + bitmap_zero(assignment->irq_bound_map, + assignment->nr_irq_vectors); + if (assignment->irq_state != MK_PCI_MSI_IDLE && + assignment->irq_state != MK_PCI_MSI_FAILED) + assignment->irq_state = MK_PCI_MSI_PREPARED; + + return disabled; +} + +static void mk_pci_release_irqs(struct mk_pci_assignment *assignment) +{ + unsigned int i; + + mk_pci_quiesce_irqs(assignment); + for (i = 0; i < assignment->nr_irq_vectors; i++) { + struct mk_pci_irq_vector *vector = &assignment->irq_vectors[i]; + + if (vector->requested) + free_irq(vector->host_irq, vector); + } + if (assignment->nr_irq_vectors) + pci_free_irq_vectors(assignment->vf); + kfree(assignment->irq_vectors); + bitmap_free(assignment->irq_bound_map); + assignment->irq_vectors = NULL; + assignment->irq_bound_map = NULL; + assignment->nr_irq_vectors = 0; + assignment->irq_msix = false; + assignment->irq_needs_reprogram = false; + assignment->irq_flr_deadline = 0; +} + +void mk_pci_quiesce_instance_irqs(struct mk_instance *instance) +{ + struct mk_pci_assignment *assignment; + unsigned int disabled = 0; + + if (!instance || instance == mk_self) + return; + + lockdep_assert_held(&instance->resource_mutex); + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + list_for_each_entry(assignment, &instance->pci_assignments, + instance_node) + disabled += mk_pci_quiesce_irqs(assignment); + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + if (disabled) + pr_info("Quiesced %u host-owned PCI IRQ vectors for instance %d\n", + disabled, instance->id); +} + +unsigned int mk_pci_sync_instance_irq_route(struct mk_instance *instance) +{ + struct mk_pci_assignment *assignment; + unsigned int requested = 0; + unsigned int i; + + lockdep_assert_held(&instance->resource_mutex); + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + list_for_each_entry(assignment, &instance->pci_assignments, + instance_node) { + for (i = 0; i < assignment->nr_irq_vectors; i++) { + struct mk_pci_irq_vector *vector = + &assignment->irq_vectors[i]; + + if (vector->requested) { + requested++; + synchronize_irq(vector->host_irq); + } + } + } + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + return requested; +} + +static bool mk_pci_generation_after(u32 generation, u32 last) +{ + return !last || (s32)(generation - last) > 0; +} + +static int mk_pci_setup_irqs(struct mk_pci_assignment *assignment, + const struct mk_pci_irq_request *request) +{ + struct mk_pci_irq_vector *vectors; + unsigned long *bound_map; + unsigned int flags; + int forwarding_cpu; + int i; + int nvec; + int ret; + + if (!request->nr_vectors) + return -EINVAL; + if (!request->lifecycle_generation) + return -EINVAL; + if (assignment->irq_epoch && + assignment->irq_epoch != request->lifecycle_epoch) + return -ESTALE; + forwarding_cpu = mk_pci_forwarding_cpu(); + if (forwarding_cpu < 0) { + pr_err("Host control CPU is unavailable for %s MSI forwarding\n", + pci_name(assignment->vf)); + return forwarding_cpu; + } + if (assignment->irq_generation == request->lifecycle_generation) { + if ((assignment->irq_state == MK_PCI_MSI_PREPARED || + assignment->irq_state == MK_PCI_MSI_COMMITTED || + assignment->irq_state == MK_PCI_MSI_ACTIVE) && + assignment->nr_irq_vectors == request->nr_vectors && + assignment->irq_msix == request->msix) + return 0; + return -ESTALE; + } + if (!mk_pci_generation_after(request->lifecycle_generation, + assignment->irq_generation)) + return -ESTALE; + if (assignment->nr_irq_vectors) + mk_pci_release_irqs(assignment); + assignment->irq_epoch = request->lifecycle_epoch; + assignment->irq_generation = request->lifecycle_generation; + assignment->irq_state = MK_PCI_MSI_FAILED; + + vectors = kcalloc(request->nr_vectors, sizeof(*vectors), GFP_KERNEL); + if (!vectors) + return -ENOMEM; + bound_map = bitmap_zalloc(request->nr_vectors, GFP_KERNEL); + if (!bound_map) { + kfree(vectors); + return -ENOMEM; + } + flags = request->msix ? PCI_IRQ_MSIX : PCI_IRQ_MSI; + nvec = pci_alloc_irq_vectors(assignment->vf, request->nr_vectors, + request->nr_vectors, flags); + if (nvec < 0) { + bitmap_free(bound_map); + kfree(vectors); + return nvec; + } + + assignment->irq_vectors = vectors; + assignment->irq_bound_map = bound_map; + assignment->nr_irq_vectors = nvec; + assignment->irq_msix = request->msix; + for (i = 0; i < nvec; i++) { + vectors[i].assignment = assignment; + vectors[i].host_irq = pci_irq_vector(assignment->vf, i); + } + for (i = 0; i < nvec; i++) { + ret = request_irq(vectors[i].host_irq, mk_pci_forward_irq, + IRQF_NO_AUTOEN | IRQF_NOBALANCING, + "multikernel-pci-forward", &vectors[i]); + if (ret) + goto err_release; + vectors[i].requested = true; + vectors[i].disabled = true; + ret = irq_set_affinity(vectors[i].host_irq, + cpumask_of(forwarding_cpu)); + if (ret) + goto err_release; + } + assignment->irq_state = MK_PCI_MSI_PREPARED; + pr_info("Allocated %d host-owned %s vectors for %s (instance %d, CPU %d)\n", + nvec, request->msix ? "MSI-X" : "MSI", + pci_name(assignment->vf), assignment->instance->id, + forwarding_cpu); + return 0; + +err_release: + pr_err("Failed to configure host-owned IRQ for %s vector %d: %d\n", + pci_name(assignment->vf), i, ret); + mk_pci_release_irqs(assignment); + return ret; +} + +static int mk_pci_bind_irq(struct mk_pci_assignment *assignment, + const struct mk_pci_irq_request *request) +{ + struct mk_pci_irq_vector *vector; + unsigned long delay; + unsigned int count; + unsigned int i; + u32 last_irq; + u32 local_irq; + + if (mk_instance_irq_route_load(assignment->instance) == + MK_PHYS_CPU_INVALID) + return -ENODEV; + if (!request->lifecycle_generation || + assignment->irq_generation != request->lifecycle_generation) + return -ESTALE; + if (assignment->irq_epoch != request->lifecycle_epoch || + (assignment->irq_state != MK_PCI_MSI_PREPARED && + assignment->irq_state != MK_PCI_MSI_ACTIVE)) + return -ESTALE; + if (!assignment->irq_vectors || + assignment->irq_msix != request->msix) + return -EINVAL; + count = request->msix ? 1 : request->nr_vectors; + if (!count || request->vector >= assignment->nr_irq_vectors || + count > assignment->nr_irq_vectors - request->vector) + return -EINVAL; + last_irq = request->local_irq + count - 1; + if (!request->local_irq || last_irq < request->local_irq) + return -EINVAL; + + for (i = 0; i < count; i++) { + vector = &assignment->irq_vectors[request->vector + i]; + local_irq = READ_ONCE(vector->local_irq); + if (!vector->requested) + return -EINVAL; + if (local_irq && local_irq != request->local_irq + i) + return -EBUSY; + } + + if (assignment->irq_needs_reprogram) { + if (time_before(jiffies, assignment->irq_flr_deadline)) { + delay = assignment->irq_flr_deadline - jiffies; + msleep(jiffies_to_msecs(delay) + 1); + } + pci_restore_msi_state(assignment->vf); + assignment->irq_needs_reprogram = false; + assignment->irq_flr_deadline = 0; + pr_info("Reprogrammed host-owned MSI state for %s during restore\n", + pci_name(assignment->vf)); + } + + for (i = 0; i < count; i++) { + vector = &assignment->irq_vectors[request->vector + i]; + WRITE_ONCE(vector->local_irq, request->local_irq + i); + } + bitmap_set(assignment->irq_bound_map, request->vector, count); + return 0; +} + +static int +mk_pci_restore_irqs_begin(struct mk_pci_assignment *assignment, + const struct mk_pci_irq_request *request) +{ + if (assignment->irq_epoch != request->lifecycle_epoch || + assignment->irq_generation != request->lifecycle_generation) + return -ESTALE; + if (assignment->irq_state == MK_PCI_MSI_PREPARED && + assignment->irq_needs_reprogram) + return 0; + if (assignment->irq_state != MK_PCI_MSI_ACTIVE) + return -ESTALE; + + mk_pci_quiesce_irqs(assignment); + assignment->irq_needs_reprogram = true; + return 0; +} + +static int mk_pci_commit_irqs(struct mk_pci_assignment *assignment, + const struct mk_pci_irq_request *request) +{ + unsigned int i; + + if (assignment->irq_epoch != request->lifecycle_epoch || + assignment->irq_generation != request->lifecycle_generation || + (assignment->irq_state != MK_PCI_MSI_PREPARED && + assignment->irq_state != MK_PCI_MSI_ACTIVE) || + !assignment->irq_bound_map || + !bitmap_full(assignment->irq_bound_map, + assignment->nr_irq_vectors)) + return -ESTALE; + + for (i = 0; i < assignment->nr_irq_vectors; i++) { + struct mk_pci_irq_vector *vector = &assignment->irq_vectors[i]; + + if (!READ_ONCE(vector->local_irq)) + return -EINVAL; + } + if (assignment->irq_state == MK_PCI_MSI_ACTIVE) + return 0; + assignment->irq_state = MK_PCI_MSI_COMMITTED; + return 0; +} + +static int mk_pci_activate_irqs(struct mk_pci_assignment *assignment, + const struct mk_pci_irq_request *request) +{ + unsigned int i; + + if (assignment->irq_epoch != request->lifecycle_epoch || + assignment->irq_generation != request->lifecycle_generation) + return -ESTALE; + if (assignment->irq_state == MK_PCI_MSI_ACTIVE) + return 0; + if (assignment->irq_state != MK_PCI_MSI_COMMITTED || + !assignment->nr_irq_vectors || !assignment->irq_bound_map || + !bitmap_full(assignment->irq_bound_map, + assignment->nr_irq_vectors)) + return -ESTALE; + for (i = 0; i < assignment->nr_irq_vectors; i++) { + struct mk_pci_irq_vector *vector = &assignment->irq_vectors[i]; + + if (!vector->requested || !READ_ONCE(vector->local_irq)) + return -EINVAL; + } + assignment->irq_state = MK_PCI_MSI_ACTIVE; + for (i = 0; i < assignment->nr_irq_vectors; i++) { + struct mk_pci_irq_vector *vector = &assignment->irq_vectors[i]; + + if (vector->disabled) { + enable_irq(vector->host_irq); + vector->disabled = false; + } + } + return 0; +} + +static int mk_pci_teardown_irqs(struct mk_pci_assignment *assignment, + const struct mk_pci_irq_request *request) +{ + if (assignment->irq_epoch && + assignment->irq_epoch != request->lifecycle_epoch) + return -ESTALE; + if (assignment->irq_generation != request->lifecycle_generation && + !mk_pci_generation_after(request->lifecycle_generation, + assignment->irq_generation)) + return -ESTALE; + + if (assignment->nr_irq_vectors) + mk_pci_release_irqs(assignment); + assignment->irq_epoch = request->lifecycle_epoch; + assignment->irq_generation = request->lifecycle_generation; + assignment->irq_state = MK_PCI_MSI_IDLE; + return 0; +} + +static bool mk_pci_is_flr_write(struct pci_dev *vf, + const struct mk_pci_cfg_request *request) +{ + u16 flr_byte; + unsigned int bit; + + if (!request->write || !pci_is_pcie(vf)) + return false; + flr_byte = pci_pcie_cap(vf) + PCI_EXP_DEVCTL + 1; + if (request->reg > flr_byte || + request->reg + request->len <= flr_byte) + return false; + bit = (flr_byte - request->reg) * 8 + 7; + return request->value & BIT(bit); +} + static bool mk_pci_request_route_stale(struct mk_instance *instance, s32 sender_instance_id, u64 lifecycle_epoch) @@ -205,6 +648,7 @@ static int mk_pci_config_access(struct mk_instance *instance, { struct mk_pci_assignment *assignment; struct pci_dev *vf; + bool flr; int ret; if (request->len != 1 && request->len != 2 && request->len != 4) @@ -225,8 +669,14 @@ static int mk_pci_config_access(struct mk_instance *instance, ret = mk_reply_begin_execute(instance, reply); if (ret) goto out; + /* Committed: complete and let publish reclaim an indeterminate timeout. */ vf = assignment->vf; + flr = mk_pci_is_flr_write(vf, request); + if (flr) { + ret = PCIBIOS_SET_FAILED; + goto out; + } if (request->write) { switch (request->len) { case 1: @@ -274,11 +724,12 @@ static void mk_pci_cfg_work_fn(struct work_struct *work) { struct mk_pci_control_work *control_work = container_of(work, struct mk_pci_control_work, work); + struct mk_pci_cfg_request *request = &control_work->request.cfg; struct mk_reply_handle reply = { - .slot = control_work->request.reply_slot, + .slot = request->reply_slot, .kind = MK_REPLY_PCI_CFG, - .request_id = control_work->request.request_id, - .generation = control_work->request.reply_generation, + .request_id = request->request_id, + .generation = request->reply_generation, }; struct mk_instance *instance; u32 value = ~0U; @@ -290,14 +741,14 @@ static void mk_pci_cfg_work_fn(struct work_struct *work) down_read(&instance->control_route_sem); status = mk_pci_request_route_stale(instance, control_work->sender_instance_id, - control_work->request.lifecycle_epoch) ? + request->lifecycle_epoch) ? -ESTALE : 0; + /* An untrusted payload ID must never select another instance's slot. */ if (status) goto unlock_route; if (mk_reply_claim(instance, &reply)) goto unlock_route; - status = mk_pci_config_access(instance, &control_work->request, - &value, &reply); + status = mk_pci_config_access(instance, request, &value, &reply); if (mk_reply_publish_route_locked(instance, &reply, status, value)) pr_warn_ratelimited("Failed to return PCI config response to instance %d\n", instance->id); @@ -308,32 +759,264 @@ static void mk_pci_cfg_work_fn(struct work_struct *work) mempool_free(control_work, mk_pci_control_pool); } +static int mk_pci_irq_access(struct mk_instance *instance, + const struct mk_pci_irq_request *request, + const struct mk_reply_handle *reply) +{ + struct mk_pci_assignment *assignment; + int ret; + + mutex_lock(&instance->resource_mutex); + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + if (request->operation != MK_PCI_IRQ_TEARDOWN && + READ_ONCE(instance->state) != MK_STATE_ACTIVE) { + ret = -ESHUTDOWN; + goto out; + } + assignment = mk_pci_find_assignment(instance, request->domain, + request->bus, request->devfn); + if (!assignment || !assignment->assigned || + !mk_pci_device_live(assignment->vf)) { + ret = -ENODEV; + goto out; + } + if (!instance->ipi_data || !request->lifecycle_epoch || + request->lifecycle_epoch != + READ_ONCE(instance->ipi_data->spawn_epoch)) { + ret = -ESTALE; + goto out; + } + ret = mk_reply_begin_execute(instance, reply); + if (ret) + goto out; + /* + * This CAS is the sole cancellation boundary. IRQ programming is + * non-cancellable once EXECUTING; a timed-out waiter moves the exact + * generation to COMMITTED for mk_reply_publish() to reclaim afterwards. + */ + + switch (request->operation) { + case MK_PCI_IRQ_SETUP: + ret = mk_pci_setup_irqs(assignment, request); + break; + case MK_PCI_IRQ_RESTORE_BEGIN: + ret = mk_pci_restore_irqs_begin(assignment, request); + break; + case MK_PCI_IRQ_BIND: + ret = mk_pci_bind_irq(assignment, request); + break; + case MK_PCI_IRQ_COMMIT: + ret = mk_pci_commit_irqs(assignment, request); + break; + case MK_PCI_IRQ_ACTIVATE: + ret = mk_pci_activate_irqs(assignment, request); + break; + case MK_PCI_IRQ_TEARDOWN: + ret = mk_pci_teardown_irqs(assignment, request); + break; + default: + ret = -EINVAL; + break; + } +out: + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + mutex_unlock(&instance->resource_mutex); + return ret; +} + +static void mk_pci_irq_work_fn(struct work_struct *work) +{ + struct mk_pci_control_work *control_work = + container_of(work, struct mk_pci_control_work, work); + struct mk_pci_irq_request *request = &control_work->request.irq; + struct mk_reply_handle reply = { + .slot = request->reply_slot, + .kind = MK_REPLY_PCI_IRQ, + .request_id = request->request_id, + .generation = request->reply_generation, + }; + struct mk_instance *instance; + s32 status; + + instance = mk_instance_find(control_work->sender_instance_id); + if (!instance) + goto out; + down_read(&instance->control_route_sem); + status = mk_pci_request_route_stale(instance, + control_work->sender_instance_id, + request->lifecycle_epoch) ? + -ESTALE : 0; + if (status) + goto unlock_route; + if (mk_reply_claim(instance, &reply)) + goto unlock_route; + status = mk_pci_irq_access(instance, request, &reply); + if (mk_reply_publish_route_locked(instance, &reply, status, 0)) + pr_warn_ratelimited("Failed to return PCI IRQ response to instance %d\n", + instance->id); +unlock_route: + up_read(&instance->control_route_sem); + mk_instance_put(instance); +out: + mempool_free(control_work, mk_pci_control_pool); +} + +static int mk_pci_reset_access(struct mk_instance *instance, + const struct mk_pci_reset_request *request, + const struct mk_reply_handle *reply) +{ + struct mk_pci_assignment *assignment; + int ret; + + mutex_lock(&instance->resource_mutex); + mutex_lock(&mk_pci_lease_mutex); + pci_lock_rescan_remove(); + if (READ_ONCE(instance->state) != MK_STATE_ACTIVE) { + ret = -ESHUTDOWN; + goto out; + } + assignment = mk_pci_find_assignment(instance, request->domain, + request->bus, request->devfn); + if (!assignment || !assignment->assigned || + !mk_pci_device_live(assignment->vf) || !assignment->vf->is_virtfn) { + ret = -ENODEV; + goto out; + } + if (!instance->ipi_data || !request->lifecycle_epoch || + request->lifecycle_epoch != + READ_ONCE(instance->ipi_data->spawn_epoch) || + !request->reset_generation || + !mk_pci_generation_after(request->reset_generation, + assignment->reset_generation)) { + ret = -ESTALE; + goto out; + } + ret = mk_reply_begin_execute(instance, reply); + if (ret) + goto out; + + /* Tombstone this serial before side effects so delayed replays reject. */ + assignment->reset_generation = request->reset_generation; + mk_pci_quiesce_irqs(assignment); + ret = pcie_reset_flr(assignment->vf, false); + if (ret) { + assignment->irq_state = MK_PCI_MSI_FAILED; + mk_pci_schedule_failure(assignment); + goto out; + } + if (assignment->nr_irq_vectors) { + assignment->irq_needs_reprogram = true; + assignment->irq_flr_deadline = + jiffies + msecs_to_jiffies(MK_PCI_FLR_SETTLE_MS); + } + pr_info("Invalidated host-owned MSI bindings for spawn-triggered FLR of %s\n", + pci_name(assignment->vf)); +out: + pci_unlock_rescan_remove(); + mutex_unlock(&mk_pci_lease_mutex); + mutex_unlock(&instance->resource_mutex); + return ret; +} + +static void mk_pci_reset_work_fn(struct work_struct *work) +{ + struct mk_pci_control_work *control_work = + container_of(work, struct mk_pci_control_work, work); + struct mk_pci_reset_request *request = &control_work->request.reset; + struct mk_reply_handle reply = { + .slot = request->reply_slot, + .kind = MK_REPLY_PCI_RESET, + .request_id = request->request_id, + .generation = request->reply_generation, + }; + struct mk_instance *instance; + s32 status; + + instance = mk_instance_find(control_work->sender_instance_id); + if (!instance) + goto out; + down_read(&instance->control_route_sem); + status = mk_pci_request_route_stale(instance, + control_work->sender_instance_id, + request->lifecycle_epoch) ? + -ESTALE : 0; + if (status) + goto unlock_route; + if (mk_reply_claim(instance, &reply)) + goto unlock_route; + status = mk_pci_reset_access(instance, request, &reply); + if (mk_reply_publish_route_locked(instance, &reply, status, 0)) + pr_warn_ratelimited("Failed to return PCI reset response to instance %d\n", + instance->id); +unlock_route: + up_read(&instance->control_route_sem); + mk_instance_put(instance); +out: + mempool_free(control_work, mk_pci_control_pool); +} + static void mk_pci_control_msg_handler(u32 msg_type, u32 subtype, void *payload, u32 payload_len, s32 sender_instance_id, void *ctx) { - struct mk_pci_control_work *work; + struct mk_pci_control_work *control_work; + size_t request_size; + work_func_t work_fn; - if (msg_type != MK_MSG_PCI || subtype != MK_PCI_CFG_REQUEST || - payload_len != sizeof(work->request)) + if (msg_type != MK_MSG_PCI) return; if (!mk_pci_control_handler_get()) return; - /* One reserve object exists for every valid outstanding reply slot. */ - work = mempool_alloc(mk_pci_control_pool, GFP_ATOMIC); - if (!work) { + switch (subtype) { + case MK_PCI_CFG_REQUEST: + request_size = sizeof(control_work->request.cfg); + work_fn = mk_pci_cfg_work_fn; + break; + case MK_PCI_IRQ_REQUEST: + request_size = sizeof(control_work->request.irq); + work_fn = mk_pci_irq_work_fn; + break; + case MK_PCI_RESET_REQUEST: + request_size = sizeof(control_work->request.reset); + work_fn = mk_pci_reset_work_fn; + break; + default: + goto out; + } + if (payload_len != request_size) + goto out; + /* + * Valid senders reserve one of MK_REPLY_SLOTS before publishing. Every + * active instance owns at least one disjoint possible CPU, so the pool + * covers the maximum number of valid requests across all instances. This + * keeps the hardirq receive path allocation-safe without changing reply + * or route validation. Duplicate traffic is outside the cooperative ABI. + */ + control_work = mempool_alloc(mk_pci_control_pool, GFP_ATOMIC); + if (!control_work) { atomic64_inc(&mk_pci_control_pool_exhausted); pr_warn_ratelimited("Multikernel PCI control work pool exhausted\n"); goto out; } - INIT_WORK(&work->work, mk_pci_cfg_work_fn); - memcpy(&work->request, payload, sizeof(work->request)); - work->sender_instance_id = sender_instance_id; - /* Payload identity is diagnostic only; endpoint provenance is authority. */ - work->request.sender_instance_id = sender_instance_id; - if (!queue_work(mk_pci_control_wq, &work->work)) - mempool_free(work, mk_pci_control_pool); + INIT_WORK(&control_work->work, work_fn); + memcpy(&control_work->request, payload, request_size); + control_work->sender_instance_id = sender_instance_id; + switch (subtype) { + case MK_PCI_CFG_REQUEST: + control_work->request.cfg.sender_instance_id = sender_instance_id; + break; + case MK_PCI_IRQ_REQUEST: + control_work->request.irq.sender_instance_id = sender_instance_id; + break; + case MK_PCI_RESET_REQUEST: + control_work->request.reset.sender_instance_id = sender_instance_id; + break; + } + if (!queue_work(mk_pci_control_wq, &control_work->work)) + mempool_free(control_work, mk_pci_control_pool); out: mk_pci_control_handler_put(); } @@ -577,6 +1260,7 @@ mk_pci_quiesce_assignment(struct mk_pci_assignment *assignment) bool transactions_drained; int ret; + mk_pci_release_irqs(assignment); if (!mk_pci_device_live(vf)) return 0; @@ -613,6 +1297,11 @@ mk_pci_reset_assignment_for_start(struct mk_pci_assignment *assignment) struct pci_dev *vf = assignment->vf; int ret; + mk_pci_release_irqs(assignment); + assignment->irq_epoch = 0; + assignment->irq_generation = 0; + assignment->reset_generation = 0; + assignment->irq_state = MK_PCI_MSI_IDLE; if (!assignment->assigned || !assignment->iommu_attached) return -EINVAL; if (!mk_pci_device_live(vf)) @@ -783,9 +1472,22 @@ static int mk_pci_iommu_assignment_probe(struct pci_dev *pdev, ret = iommu_attach_group(assignment->iommu_domain, assignment->iommu_group); - if (ret) + if (ret) { + iommu_device_release_dma_owner(&assignment->vf->dev); + assignment->iommu_dma_owner = false; return ret; + } assignment->iommu_attached = true; + ret = pci_enable_device(pdev); + if (ret) { + iommu_detach_group(assignment->iommu_domain, + assignment->iommu_group); + assignment->iommu_attached = false; + iommu_device_release_dma_owner(&assignment->vf->dev); + assignment->iommu_dma_owner = false; + return ret; + } + assignment->device_enabled = true; pr_info("Attached %s to host-owned IOMMU domain for instance %d\n", pci_name(assignment->vf), assignment->instance->id); return 0; @@ -800,6 +1502,10 @@ static void mk_pci_iommu_assignment_remove(struct pci_dev *pdev) return; if (READ_ONCE(assignment->expected_unbind)) { + if (assignment->device_enabled) { + pci_disable_device(pdev); + assignment->device_enabled = false; + } pci_set_drvdata(pdev, NULL); return; } @@ -812,6 +1518,10 @@ static void mk_pci_iommu_assignment_remove(struct pci_dev *pdev) } else { mk_pci_iommu_deactivate_assignment(assignment); } + if (assignment->device_enabled) { + pci_disable_device(pdev); + assignment->device_enabled = false; + } pci_set_drvdata(pdev, NULL); } From 69f1e8167c456f7d8cb82fe282415ede5ca07b7e Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Tue, 11 Aug 2026 05:10:33 +0300 Subject: [PATCH 15/24] multikernel: add lossless pending IRQ mailboxes A bounded shared ring cannot guarantee delivery when a host interrupt arrives in hardirq context. Record assigned interrupts in preallocated per-instance pending slots and use the IPI only as a doorbell. Protect each slot with spawn epoch, lifecycle generation, and an atomic pending, masked, and consuming token. Coalesce while masked and retry lost doorbells until the guest drains the slot. Bump the exact transport ABI to version 7. Signed-off-by: Nikolay Nikolaev --- Documentation/multikernel/usage.rst | 3 + arch/x86/multikernel/pci.c | 380 +++++++++++++++++--- include/linux/multikernel.h | 86 +++++ kernel/multikernel/core.c | 26 +- kernel/multikernel/internal.h | 3 +- kernel/multikernel/ipi.c | 3 + kernel/multikernel/pci.c | 518 ++++++++++++++++++++++++---- 7 files changed, 886 insertions(+), 133 deletions(-) diff --git a/Documentation/multikernel/usage.rst b/Documentation/multikernel/usage.rst index 234c5a069d2eec..bf876c209604a5 100644 --- a/Documentation/multikernel/usage.rst +++ b/Documentation/multikernel/usage.rst @@ -99,3 +99,6 @@ Restrictions - Rollback (``rmdir`` on a transaction) cannot destroy a running instance. - Logical CPU 0 handles assigned-device MSI forwarding and must remain online while those devices are active. +- Assigned MSI events are recorded in per-instance shared mailboxes before a + doorbell is sent. Masked events remain pending and lost doorbells are retried; + each spawn drains only its own parent-link mailbox. diff --git a/arch/x86/multikernel/pci.c b/arch/x86/multikernel/pci.c index 7a5b5049b11b12..f39532f020f579 100644 --- a/arch/x86/multikernel/pci.c +++ b/arch/x86/multikernel/pci.c @@ -7,6 +7,7 @@ */ #include +#include #include #include #include @@ -27,14 +28,22 @@ static atomic64_t mk_pci_cfg_total_ns = ATOMIC64_INIT(0); static atomic64_t mk_pci_cfg_max_ns = ATOMIC64_INIT(0); #define MK_PCI_RESET_TIMEOUT_MS 70000 #ifdef CONFIG_PCI_MSI +static void mk_pci_forward_irq_set_mask(struct irq_data *data, bool masked); +static void mk_pci_irq_mailbox_requeue(struct mk_irq_mailbox *mailbox, + unsigned int slot); -static bool mk_pci_message_from_host(s32 sender_instance_id) +static void mk_pci_forward_irq_noop(struct irq_data *data) { - return host_instance && sender_instance_id == host_instance->id; } -static void mk_pci_forward_irq_noop(struct irq_data *data) +static void mk_pci_forward_irq_mask(struct irq_data *data) +{ + mk_pci_forward_irq_set_mask(data, true); +} + +static void mk_pci_forward_irq_unmask(struct irq_data *data) { + mk_pci_forward_irq_set_mask(data, false); } static void mk_pci_forward_irq_write_msg(struct irq_data *data, @@ -46,8 +55,8 @@ static struct irq_chip mk_pci_forward_irq_chip = { .name = "multikernel-pci-forward", .irq_ack = mk_pci_forward_irq_noop, /* Host process-context lifecycle owns physical mask state. */ - .irq_mask = mk_pci_forward_irq_noop, - .irq_unmask = mk_pci_forward_irq_noop, + .irq_mask = mk_pci_forward_irq_mask, + .irq_unmask = mk_pci_forward_irq_unmask, .irq_write_msi_msg = mk_pci_forward_irq_write_msg, }; @@ -60,10 +69,11 @@ static void mk_pci_bind_local_irqs(unsigned int irq, unsigned int count) handle_edge_irq); } -static bool mk_pci_forward_irq_matches(const struct mk_io_irq_payload *irq, +static bool mk_pci_forward_irq_matches(u32 irq_number, u32 vector, + u32 device_id, struct irq_data **irq_data) { - struct irq_data *data = irq_get_irq_data(irq->irq_number); + struct irq_data *data = irq_get_irq_data(irq_number); struct msi_desc *desc; struct pci_dev *dev; unsigned int offset; @@ -71,58 +81,251 @@ static bool mk_pci_forward_irq_matches(const struct mk_io_irq_payload *irq, if (!data) return false; desc = irq_data_get_msi_desc(data); - if (!desc || irq->vector < desc->msi_index) + if (!desc || vector < desc->msi_index) return false; dev = msi_desc_to_pci_dev(desc); - offset = irq->vector - desc->msi_index; - if (offset >= desc->nvec_used || desc->irq + offset != irq->irq_number || - pci_domain_nr(dev->bus) != MK_PCI_IRQ_ID_DOMAIN(irq->device_id) || - dev->bus->number != MK_PCI_IRQ_ID_BUS(irq->device_id) || - dev->devfn != MK_PCI_IRQ_ID_DEVFN(irq->device_id)) + offset = vector - desc->msi_index; + if (offset >= desc->nvec_used || desc->irq + offset != irq_number || + pci_domain_nr(dev->bus) != MK_PCI_IRQ_ID_DOMAIN(device_id) || + dev->bus->number != MK_PCI_IRQ_ID_BUS(device_id) || + dev->devfn != MK_PCI_IRQ_ID_DEVFN(device_id)) return false; *irq_data = data; return true; } -static void mk_pci_irq_forward_handler(u32 msg_type, u32 subtype, - void *payload, u32 payload_len, - s32 sender_instance_id, void *ctx) +static void mk_pci_forward_irq_set_mask(struct irq_data *data, bool masked) { - struct mk_io_irq_payload *irq = payload; - struct irq_data *irq_data; + struct msi_desc *desc = irq_data_get_msi_desc(data); + struct mk_irq_mailbox *mailbox; struct pci_dev *dev; + u32 device_id; + u32 generation; + u32 vector; + unsigned int slot; + u64 epoch; - if (msg_type != MK_MSG_IO || subtype != MK_IO_IRQ_FORWARD || - payload_len != sizeof(*irq) || - !mk_pci_message_from_host(sender_instance_id)) + if (!desc || !mk_self || !mk_self->ipi_data) + return; + dev = msi_desc_to_pci_dev(desc); + generation = READ_ONCE(dev->multikernel_msi_generation); + epoch = READ_ONCE(mk_self->ipi_data->spawn_epoch); + device_id = MK_PCI_IRQ_ID(pci_domain_nr(dev->bus), dev->bus->number, + dev->devfn); + vector = desc->msi_index + data->irq - desc->irq; + mailbox = &mk_self->ipi_data->irq_mailbox; + for (slot = 0; slot < MK_IRQ_MAILBOX_SLOTS; slot++) { + struct mk_irq_mailbox_entry *entry = &mailbox->entries[slot]; + u64 old, new; + u32 slot_generation; + + old = atomic64_read_acquire(&entry->pending_generation); + if (!mk_irq_mailbox_generation(old) || + READ_ONCE(entry->lifecycle_epoch) != epoch || + READ_ONCE(entry->lifecycle_generation) != generation || + READ_ONCE(entry->device_id) != device_id || + READ_ONCE(entry->local_irq) != data->irq || + READ_ONCE(entry->vector) != vector) + continue; + slot_generation = mk_irq_mailbox_generation(old); + for (;;) { + new = masked ? old | MK_IRQ_MAILBOX_MASKED : + old & ~MK_IRQ_MAILBOX_MASKED; + if (new == old || + atomic64_cmpxchg(&entry->pending_generation, + old, new) == old) + break; + old = atomic64_read_acquire(&entry->pending_generation); + if (mk_irq_mailbox_generation(old) != slot_generation) + return; + } + if (!masked && mk_irq_mailbox_pending(new)) { + mk_phys_cpu_t target; + + mk_pci_irq_mailbox_requeue(mailbox, slot); + target = arch_cpu_physical_id(smp_processor_id()); + if (target != MK_PHYS_CPU_INVALID) + mk_arch_send_ipi(target); + } + return; + } +} + +static void mk_pci_irq_mailbox_requeue(struct mk_irq_mailbox *mailbox, + unsigned int slot) +{ + atomic64_or(BIT_ULL(slot & 63), + &mailbox->pending_bitmap[slot / 64]); +} + +static void mk_pci_irq_mailbox_drain_slot(struct mk_shared_data *shared, + unsigned int slot) +{ + struct mk_irq_mailbox *mailbox = &shared->irq_mailbox; + struct mk_irq_mailbox_entry *entry = &mailbox->entries[slot]; + struct irq_data *irq_data; + struct pci_dev *dev; + u64 lifecycle_epoch; + u64 token, base, claim; + u32 lifecycle_generation; + u32 pending; + u32 device_id; + u32 local_irq; + u16 vector; + + token = atomic64_read_acquire(&entry->pending_generation); + pending = mk_irq_mailbox_pending(token); + if (!mk_irq_mailbox_generation(token) || !pending) + return; + if (mk_irq_mailbox_consuming(token)) { + mk_pci_irq_mailbox_requeue(mailbox, slot); return; - if (!mk_pci_forward_irq_matches(irq, &irq_data)) { - pr_warn_ratelimited("Rejected host-forwarded PCI IRQ %u with unmatched identity %#x vector %u\n", - irq->irq_number, irq->device_id, - irq->vector); + } + lifecycle_epoch = READ_ONCE(entry->lifecycle_epoch); + lifecycle_generation = READ_ONCE(entry->lifecycle_generation); + device_id = READ_ONCE(entry->device_id); + local_irq = READ_ONCE(entry->local_irq); + vector = READ_ONCE(entry->vector); + base = token & ~MK_IRQ_MAILBOX_PENDING_MASK; + + if (!local_irq || !mk_self || !mk_self->ipi_data || + lifecycle_epoch != READ_ONCE(shared->spawn_epoch)) + goto stale; + if (mk_irq_mailbox_masked(token)) { + atomic_inc(&mailbox->masked_deferred); + mk_pci_irq_mailbox_requeue(mailbox, slot); + return; + } + claim = base | MK_IRQ_MAILBOX_CONSUMING; + if (atomic64_cmpxchg_acquire(&entry->pending_generation, + token, claim) != token) { + mk_pci_irq_mailbox_requeue(mailbox, slot); return; } + token = atomic64_read_acquire(&entry->pending_generation); + if (mk_irq_mailbox_generation(token) != + mk_irq_mailbox_generation(claim)) + goto claimed_stale; + if (mk_irq_mailbox_masked(token)) { + atomic_inc(&mailbox->masked_deferred); + goto claimed_defer; + } + if (!mk_pci_forward_irq_matches(local_irq, vector, device_id, + &irq_data)) + goto claimed_stale; dev = msi_desc_to_pci_dev(irq_data_get_msi_desc(irq_data)); - if (READ_ONCE(dev->multikernel_msi_state) != MK_PCI_MSI_ACTIVE || - irq->lifecycle_generation != + if (lifecycle_epoch != READ_ONCE(entry->lifecycle_epoch) || + lifecycle_generation != + READ_ONCE(entry->lifecycle_generation) || + device_id != READ_ONCE(entry->device_id) || + local_irq != READ_ONCE(entry->local_irq) || + vector != READ_ONCE(entry->vector) || + READ_ONCE(dev->multikernel_msi_state) != MK_PCI_MSI_ACTIVE || + lifecycle_generation != READ_ONCE(dev->multikernel_msi_generation) || - !mk_self->ipi_data || - irq->lifecycle_epoch != - READ_ONCE(mk_self->ipi_data->spawn_epoch)) + irq_data_get_irq_chip(irq_data) != &mk_pci_forward_irq_chip) + goto claimed_stale; + if (pending > 1) + atomic_add(pending - 1, &mailbox->coalesced); + if (generic_handle_irq_safe(local_irq)) + atomic_inc(&mailbox->dispatch_failed); + + for (;;) { + u64 new; + + token = atomic64_read_acquire(&entry->pending_generation); + if (mk_irq_mailbox_generation(token) != + mk_irq_mailbox_generation(claim) || + !mk_irq_mailbox_consuming(token)) + return; + new = token & ~MK_IRQ_MAILBOX_CONSUMING; + if (atomic64_cmpxchg_release(&entry->pending_generation, + token, new) != token) + continue; + if (mk_irq_mailbox_pending(new)) + mk_pci_irq_mailbox_requeue(mailbox, slot); return; + } - if (irq_data_get_irq_chip(irq_data) != &mk_pci_forward_irq_chip) { - pr_warn_ratelimited("Rejected host-forwarded PCI IRQ %u for %s vector %u before local binding\n", - irq->irq_number, pci_name(dev), - irq->vector); +claimed_defer: + for (;;) { + u32 new_pending; + u64 new; + + token = atomic64_read_acquire(&entry->pending_generation); + if (mk_irq_mailbox_generation(token) != + mk_irq_mailbox_generation(claim) || + !mk_irq_mailbox_consuming(token)) + return; + new_pending = mk_irq_mailbox_pending(token); + if (new_pending < MK_IRQ_MAILBOX_PENDING_MASK) + new_pending++; + new = mk_irq_mailbox_token(mk_irq_mailbox_generation(token), + new_pending) | + (token & MK_IRQ_MAILBOX_MASKED); + if (atomic64_cmpxchg_release(&entry->pending_generation, + token, new) != token) + continue; + mk_pci_irq_mailbox_requeue(mailbox, slot); + if (!mk_irq_mailbox_masked(new)) { + mk_phys_cpu_t target = + arch_cpu_physical_id(smp_processor_id()); + + if (target != MK_PHYS_CPU_INVALID) + mk_arch_send_ipi(target); + } return; } - if (generic_handle_irq_safe(irq->irq_number)) - pr_warn_ratelimited("Failed to dispatch host-forwarded PCI IRQ %u\n", - irq->irq_number); +claimed_stale: + atomic_inc(&mailbox->stale); + for (;;) { + u64 new; + + token = atomic64_read_acquire(&entry->pending_generation); + if (mk_irq_mailbox_generation(token) != + mk_irq_mailbox_generation(claim) || + !mk_irq_mailbox_consuming(token)) + return; + new = token & ~MK_IRQ_MAILBOX_CONSUMING; + if (atomic64_cmpxchg_release(&entry->pending_generation, + token, new) != token) + continue; + if (mk_irq_mailbox_pending(new)) + mk_pci_irq_mailbox_requeue(mailbox, slot); + return; + } + return; + +stale: + if (atomic64_cmpxchg_acquire(&entry->pending_generation, + token, base) != token) + mk_pci_irq_mailbox_requeue(mailbox, slot); + atomic_inc(&mailbox->stale); +} + +void mk_pci_irq_mailbox_drain(struct mk_shared_data *shared) +{ + unsigned int word; + + if (!shared) + return; + for (word = 0; word < MK_IRQ_MAILBOX_WORDS; word++) { + atomic64_t *pending_bitmap; + unsigned long bits; + + pending_bitmap = &shared->irq_mailbox.pending_bitmap[word]; + bits = atomic64_xchg_acquire(pending_bitmap, 0); + + while (bits) { + unsigned int bit = __ffs(bits); + + bits &= bits - 1; + mk_pci_irq_mailbox_drain_slot(shared, word * 64 + bit); + } + } } static int mk_pci_send_irq_request(struct mk_pci_irq_request *request) @@ -228,10 +431,26 @@ static int mk_pci_msi_bind(struct pci_dev *dev, unsigned int index, .local_irq = irq, .lifecycle_generation = generation, }; + unsigned int count = msix ? 1 : nvec; + unsigned int i; + int ret; /* The local descriptor must be dispatchable before the host unmasks. */ - mk_pci_bind_local_irqs(irq, msix ? 1 : nvec); - return mk_pci_send_irq_request(&request); + mk_pci_bind_local_irqs(irq, count); + ret = mk_pci_send_irq_request(&request); + if (ret) + return ret; + /* Publish the irqdesc's initial logical mask state into the token. */ + for (i = 0; i < count; i++) { + struct irq_data *data = irq_get_irq_data(irq + i); + + if (!data) + return -EINVAL; + mk_pci_forward_irq_set_mask(data, + irqd_irq_disabled(data) || + irqd_irq_masked(data)); + } + return 0; } static int mk_pci_msi_teardown_generation(struct pci_dev *dev, u32 generation) @@ -311,6 +530,69 @@ static int mk_pci_msi_bind_all(struct pci_dev *dev, u32 generation) return mk_pci_msi_commit(dev, generation); } +static int mk_pci_msi_mask_mailbox(struct pci_dev *dev) +{ + struct msi_desc *desc; + unsigned long deadline; + u64 epoch; + u32 device_id; + u32 generation; + unsigned int slot; + + msi_for_each_desc(desc, &dev->dev, MSI_DESC_ALL) { + unsigned int count = desc->pci.msi_attrib.is_msix ? + 1 : desc->nvec_used; + unsigned int i; + + for (i = 0; i < count; i++) { + struct irq_data *data = irq_get_irq_data(desc->irq + i); + + if (data) + mk_pci_forward_irq_set_mask(data, true); + } + } + + epoch = READ_ONCE(mk_self->ipi_data->spawn_epoch); + generation = READ_ONCE(dev->multikernel_msi_generation); + device_id = MK_PCI_IRQ_ID(pci_domain_nr(dev->bus), dev->bus->number, + dev->devfn); + deadline = jiffies + msecs_to_jiffies(1000); + for (;;) { + bool consuming = false; + + for (slot = 0; slot < MK_IRQ_MAILBOX_SLOTS; slot++) { + struct mk_irq_mailbox_entry *entry = + &mk_self->ipi_data->irq_mailbox.entries[slot]; + u64 token; + + token = atomic64_read_acquire(&entry->pending_generation); + + if (mk_irq_mailbox_generation(token) && + mk_irq_mailbox_consuming(token) && + READ_ONCE(entry->lifecycle_epoch) == epoch && + READ_ONCE(entry->lifecycle_generation) == generation && + READ_ONCE(entry->device_id) == device_id) { + consuming = true; + break; + } + } + if (!consuming) + break; + if (time_after_eq(jiffies, deadline)) + return -ETIMEDOUT; + usleep_range(50, 100); + } + msi_for_each_desc(desc, &dev->dev, MSI_DESC_ALL) { + unsigned int count = desc->pci.msi_attrib.is_msix ? + 1 : desc->nvec_used; + unsigned int i; + + for (i = 0; i < count; i++) + synchronize_irq(desc->irq + i); + } + return 0; +} + int mk_pci_msi_activate(struct pci_dev *dev) { u32 generation; @@ -336,7 +618,9 @@ int mk_pci_msi_activate(struct pci_dev *dev) WRITE_ONCE(dev->multikernel_msi_state, MK_PCI_MSI_ACTIVE); ret = mk_pci_msi_host_activate(dev, generation); if (ret) { - cleanup_ret = mk_pci_msi_teardown_generation(dev, generation); + cleanup_ret = mk_pci_msi_mask_mailbox(dev); + if (!cleanup_ret) + cleanup_ret = mk_pci_msi_teardown_generation(dev, generation); WRITE_ONCE(dev->multikernel_msi_state, cleanup_ret ? MK_PCI_MSI_FAILED : MK_PCI_MSI_IDLE); return ret; @@ -355,6 +639,11 @@ int mk_pci_msi_restore(struct pci_dev *dev) if (READ_ONCE(dev->multikernel_msi_state) != MK_PCI_MSI_ACTIVE) return -EIO; generation = READ_ONCE(dev->multikernel_msi_generation); + ret = mk_pci_msi_mask_mailbox(dev); + if (ret) { + WRITE_ONCE(dev->multikernel_msi_state, MK_PCI_MSI_FAILED); + return ret; + } ret = mk_pci_msi_restore_begin(dev, generation); if (!ret) ret = mk_pci_msi_bind_all(dev, generation); @@ -381,7 +670,9 @@ int mk_pci_msi_teardown(struct pci_dev *dev) if (READ_ONCE(dev->multikernel_msi_state) == MK_PCI_MSI_IDLE) return 0; generation = READ_ONCE(dev->multikernel_msi_generation); - ret = mk_pci_msi_teardown_generation(dev, generation); + ret = mk_pci_msi_mask_mailbox(dev); + if (!ret) + ret = mk_pci_msi_teardown_generation(dev, generation); WRITE_ONCE(dev->multikernel_msi_state, ret ? MK_PCI_MSI_FAILED : MK_PCI_MSI_IDLE); return ret; @@ -576,13 +867,6 @@ static int __init x86_multikernel_pci_arch_init(void) { if (!mk_self || !host_instance || !mk_self->pci_devices_valid) return 0; -#ifdef CONFIG_PCI_MSI - if (mk_register_msg_handler(MK_MSG_IO, mk_pci_irq_forward_handler, - NULL)) { - pr_err("Multikernel failed to register PCI IRQ forwarding handler\n"); - return 0; - } -#endif raw_pci_ops = &mk_pci_filtered_raw_ops; raw_pci_ext_ops = &mk_pci_filtered_raw_ops; diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index f95612a38e6050..c31ac674ed98cc 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -87,6 +87,13 @@ void mk_cpu_transaction_unlock(void); #define MK_REPLY_SLOTS 16 #define MK_REPLY_STATE_BITS 3 +#define MK_IRQ_MAILBOX_SLOTS 256 +#define MK_IRQ_MAILBOX_WORDS (MK_IRQ_MAILBOX_SLOTS / 64) +#define MK_IRQ_MAILBOX_SLOT_INVALID ((u32)~0U) +#define MK_IRQ_MAILBOX_PENDING_MASK 0x3fffffffULL +#define MK_IRQ_MAILBOX_CONSUMING 0x40000000ULL +#define MK_IRQ_MAILBOX_MASKED 0x80000000ULL +#define MK_IRQ_MAILBOX_GENERATION_SHIFT 32 enum mk_reply_state { MK_REPLY_FREE = 0, @@ -144,6 +151,53 @@ struct mk_ipi_ring { struct mk_ipi_data entries[MK_IPI_RING_SIZE]; /* Ring buffer entries */ }; +struct mk_irq_mailbox_entry { + atomic64_t pending_generation; + u64 lifecycle_epoch; + u32 lifecycle_generation; + u32 device_id; + u32 local_irq; + u16 vector; + u16 reserved; +}; + +struct mk_irq_mailbox { + atomic64_t pending_bitmap[MK_IRQ_MAILBOX_WORDS]; + struct mk_irq_mailbox_entry entries[MK_IRQ_MAILBOX_SLOTS]; + atomic_t next_generation; + atomic_t recorded; + atomic_t coalesced; + atomic_t masked_deferred; + atomic_t stale; + atomic_t dispatch_failed; + atomic_t saturated; +}; + +static inline u64 mk_irq_mailbox_token(u32 generation, u32 pending) +{ + return (u64)generation << MK_IRQ_MAILBOX_GENERATION_SHIFT | pending; +} + +static inline u32 mk_irq_mailbox_generation(u64 token) +{ + return token >> MK_IRQ_MAILBOX_GENERATION_SHIFT; +} + +static inline u32 mk_irq_mailbox_pending(u64 token) +{ + return token & MK_IRQ_MAILBOX_PENDING_MASK; +} + +static inline bool mk_irq_mailbox_masked(u64 token) +{ + return token & MK_IRQ_MAILBOX_MASKED; +} + +static inline bool mk_irq_mailbox_consuming(u64 token) +{ + return token & MK_IRQ_MAILBOX_CONSUMING; +} + /* The spawn boot tree's fixed budget, shared by allocator and writer */ #define MK_MANIFEST_SIZE SZ_256K /* Room the manifest leaves for a user-provided host tree */ @@ -184,6 +238,7 @@ struct mk_shared_data { u64 spawn_epoch; /* Generation-tagged synchronous replies, independent of ring progress. */ struct mk_reply_table replies; + struct mk_irq_mailbox irq_mailbox; }; static inline void mk_reply_table_reset(struct mk_reply_table *table) @@ -212,6 +267,32 @@ static inline void mk_ipi_ring_reset(struct mk_ipi_ring *ring) WRITE_ONCE(ring->entries[i].ready, 0); } +static inline void mk_irq_mailbox_reset(struct mk_irq_mailbox *mailbox) +{ + unsigned int i; + + for (i = 0; i < MK_IRQ_MAILBOX_WORDS; i++) + atomic64_set(&mailbox->pending_bitmap[i], 0); + for (i = 0; i < MK_IRQ_MAILBOX_SLOTS; i++) { + struct mk_irq_mailbox_entry *entry = &mailbox->entries[i]; + + atomic64_set(&entry->pending_generation, 0); + WRITE_ONCE(entry->lifecycle_epoch, 0); + WRITE_ONCE(entry->lifecycle_generation, 0); + WRITE_ONCE(entry->device_id, 0); + WRITE_ONCE(entry->local_irq, 0); + WRITE_ONCE(entry->vector, 0); + WRITE_ONCE(entry->reserved, 0); + } + atomic_set(&mailbox->next_generation, 0); + atomic_set(&mailbox->recorded, 0); + atomic_set(&mailbox->coalesced, 0); + atomic_set(&mailbox->masked_deferred, 0); + atomic_set(&mailbox->stale, 0); + atomic_set(&mailbox->dispatch_failed, 0); + atomic_set(&mailbox->saturated, 0); +} + static inline void mk_shared_data_reset(struct mk_shared_data *shared) { unsigned int i; @@ -222,6 +303,7 @@ static inline void mk_shared_data_reset(struct mk_shared_data *shared) for (i = 0; i < MK_PARKED_MAX; i++) WRITE_ONCE(shared->parked[i], 0); mk_reply_table_reset(&shared->replies); + mk_irq_mailbox_reset(&shared->irq_mailbox); } struct mk_ipi_endpoint { @@ -873,6 +955,7 @@ struct mk_instance { /* Pins the CPU selected for control messages and forwarded IRQs. */ struct rw_semaphore control_route_sem; mk_phys_cpu_t irq_route_cpu; + struct delayed_work irq_retry_work; /* PCI device resources */ struct list_head pci_devices; /* List of struct mk_pci_device */ @@ -1202,12 +1285,14 @@ int mk_pci_msi_prepare(struct pci_dev *dev, int nvec, int type); int mk_pci_msi_activate(struct pci_dev *dev); int mk_pci_msi_restore(struct pci_dev *dev); int mk_pci_msi_teardown(struct pci_dev *dev); +void mk_pci_irq_mailbox_drain(struct mk_shared_data *shared); #else static inline bool mk_pci_msi_controlled(struct pci_dev *dev) { return false; } static inline int mk_pci_msi_prepare(struct pci_dev *dev, int nvec, int type) { return 0; } static inline int mk_pci_msi_activate(struct pci_dev *dev) { return 0; } static inline int mk_pci_msi_restore(struct pci_dev *dev) { return 0; } static inline int mk_pci_msi_teardown(struct pci_dev *dev) { return 0; } +static inline void mk_pci_irq_mailbox_drain(struct mk_shared_data *shared) { } #endif bool mk_platform_device_allowed(const char *name, const char *hid); @@ -1242,6 +1327,7 @@ static inline int mk_pci_msi_prepare(struct pci_dev *dev, int nvec, int type) { static inline int mk_pci_msi_activate(struct pci_dev *dev) { return 0; } static inline int mk_pci_msi_restore(struct pci_dev *dev) { return 0; } static inline int mk_pci_msi_teardown(struct pci_dev *dev) { return 0; } +static inline void mk_pci_irq_mailbox_drain(struct mk_shared_data *shared) { } static inline int multikernel_force_halt_by_id(int mk_id) { diff --git a/kernel/multikernel/core.c b/kernel/multikernel/core.c index ea69e3769c626a..74d866085431d6 100644 --- a/kernel/multikernel/core.c +++ b/kernel/multikernel/core.c @@ -1593,18 +1593,25 @@ static void mk_shutdown_work_fn(struct work_struct *work) * it corrupts the single-producer mailbox. The kexec path confirms the * CPUs are parked before it rewrites the image. */ -static void mk_instance_settle_halted(struct mk_instance *instance) +static int mk_instance_settle_halted(struct mk_instance *instance) { + int ret; + pr_info("Instance %d (%s) halted, CPUs parking in pool\n", instance->id, instance->name); + ret = mk_instance_confirm_parked(instance); + if (ret) + return ret; mk_cpu_transaction_lock(); down_write(&instance->control_route_sem); mutex_lock(&instance->resource_mutex); + ret = mk_pci_quiesce_instance_irqs(instance, true); mk_instance_irq_route_store(instance, MK_PHYS_CPU_INVALID); - mk_instance_set_state(instance, MK_STATE_LOADED); + mk_instance_set_state(instance, ret ? MK_STATE_FAILED : MK_STATE_LOADED); mutex_unlock(&instance->resource_mutex); up_write(&instance->control_route_sem); mk_cpu_transaction_unlock(); + return ret; } struct mk_halted_work { @@ -1620,7 +1627,9 @@ static void mk_halted_work_fn(struct work_struct *work) instance = mk_instance_find(aw->instance_id); if (instance) { - mk_instance_settle_halted(instance); + if (mk_instance_settle_halted(instance)) + pr_err("Instance %d halted but could not be made reusable\n", + instance->id); mk_instance_put(instance); } else { pr_warn("Shutdown ACK from unknown instance %d\n", @@ -1749,12 +1758,9 @@ int multikernel_halt_by_id(int mk_id) ret = mk_msg_pending_wait(pending, 30000); if (ret == 0) { - if (mk_instance_confirm_parked(instance)) - pr_warn("Multikernel instance %d halted with CPUs unaccounted for\n", - mk_id); - - mk_instance_set_state(instance, MK_STATE_LOADED); - pr_info("Multikernel instance %d halted (graceful)\n", mk_id); + ret = mk_instance_settle_halted(instance); + if (!ret) + pr_info("Multikernel instance %d halted (graceful)\n", mk_id); } mk_instance_put(instance); @@ -1967,7 +1973,7 @@ static int __mk_instance_force_halt(struct mk_instance *instance, pr_err("Instance %d CPUs did not park after force halt: %d\n", instance->id, ret); else - mk_instance_settle_halted(instance); + ret = mk_instance_settle_halted(instance); } mk_cpu_set_free(targets); diff --git a/kernel/multikernel/internal.h b/kernel/multikernel/internal.h index f2199d8cd2c723..19f865c3b79068 100644 --- a/kernel/multikernel/internal.h +++ b/kernel/multikernel/internal.h @@ -79,7 +79,8 @@ int mk_pci_assign_device(struct mk_instance *instance, u16 domain, u8 bus, int mk_pci_unassign_device(struct mk_instance *instance, u16 domain, u8 bus, u8 devfn); int mk_pci_release_assignments(struct mk_instance *instance); -void mk_pci_quiesce_instance_irqs(struct mk_instance *instance); +int mk_pci_quiesce_instance_irqs(struct mk_instance *instance, + bool parked_force); unsigned int mk_pci_sync_instance_irq_route(struct mk_instance *instance); /* overlay.c */ extern struct kernfs_node *mk_overlay_root_kn; diff --git a/kernel/multikernel/ipi.c b/kernel/multikernel/ipi.c index 26336ff3eed90e..c93a9f46a5b8f8 100644 --- a/kernel/multikernel/ipi.c +++ b/kernel/multikernel/ipi.c @@ -695,6 +695,9 @@ static void mk_ipi_drain_all(void) if (!READ_ONCE(mk_handlers_ready)) return; + /* A kernel consumes only its own parent-link IRQ mailbox. */ + if (mk_self && mk_self->ipi_data) + mk_pci_irq_mailbox_drain(mk_self->ipi_data); rcu_read_lock(); list_for_each_entry_rcu(endpoint, &mk_ipi_endpoints, rx_node) mk_ipi_drain_endpoint(endpoint); diff --git a/kernel/multikernel/pci.c b/kernel/multikernel/pci.c index 8418057eb3e759..73fd839c317400 100644 --- a/kernel/multikernel/pci.c +++ b/kernel/multikernel/pci.c @@ -5,6 +5,12 @@ * Keeps assigned-device discovery, identity presentation, bridge traversal, * and resource restoration independent from the manifest that populated * the current instance. + * + * Configuration-space filtering constrains normal PCI access by a cooperative + * spawn kernel. It is not a security boundary: a privileged kernel can issue + * configuration cycles or map physical configuration windows directly. The + * host-owned IOMMU domain separately constrains DMA initiated by an assigned + * device. */ #include @@ -26,11 +32,19 @@ struct mk_pci_assignment; #define MK_PCI_FLR_SETTLE_MS 100 +#define MK_PCI_MAILBOX_QUIESCE_MS 1000 + +static unsigned long mk_pci_mailbox_deadline(void) +{ + return jiffies + msecs_to_jiffies(MK_PCI_MAILBOX_QUIESCE_MS); +} struct mk_pci_irq_vector { struct mk_pci_assignment *assignment; unsigned int host_irq; u32 local_irq; + u32 mailbox_slot; + u32 mailbox_generation; atomic64_t forwarded; bool requested; bool disabled; @@ -62,6 +76,7 @@ struct mk_pci_assignment { u8 irq_state; bool irq_msix; bool irq_needs_reprogram; + bool irq_mailbox_failed; unsigned long irq_flr_deadline; bool device_enabled; struct work_struct failure_work; @@ -226,45 +241,249 @@ mk_pci_find_assignment(struct mk_instance *instance, u16 domain, u8 bus, return NULL; } +static void mk_pci_mailbox_clear_bit(struct mk_irq_mailbox *mailbox, + unsigned int slot) +{ + atomic64_andnot(BIT_ULL(slot & 63), + &mailbox->pending_bitmap[slot / 64]); +} + +static bool mk_pci_mailbox_quiesce(struct mk_pci_irq_vector *vector, + bool parked_force, + unsigned long deadline) +{ + struct mk_shared_data *shared = vector->assignment->instance->ipi_data; + struct mk_irq_mailbox *mailbox; + struct mk_irq_mailbox_entry *entry; + u64 base, token; + + if (!shared || vector->mailbox_slot == MK_IRQ_MAILBOX_SLOT_INVALID || + !vector->mailbox_generation) + return true; + mailbox = &shared->irq_mailbox; + entry = &mailbox->entries[vector->mailbox_slot]; + base = mk_irq_mailbox_token(vector->mailbox_generation, 0) | + MK_IRQ_MAILBOX_MASKED; + for (;;) { + token = atomic64_read_acquire(&entry->pending_generation); + if (mk_irq_mailbox_generation(token) != + vector->mailbox_generation) { + atomic_inc(&mailbox->stale); + return false; + } + if (mk_irq_mailbox_consuming(token)) { + if (parked_force) { + if (atomic64_cmpxchg_release(&entry->pending_generation, + token, base) == token) + break; + continue; + } + if (!mk_irq_mailbox_masked(token)) + atomic64_cmpxchg(&entry->pending_generation, + token, + token | MK_IRQ_MAILBOX_MASKED); + if (time_after_eq(jiffies, deadline)) + return false; + usleep_range(50, 100); + continue; + } + if (atomic64_cmpxchg_release(&entry->pending_generation, + token, base) == token) + break; + cpu_relax(); + } + mk_pci_mailbox_clear_bit(mailbox, vector->mailbox_slot); + WRITE_ONCE(entry->local_irq, 0); + return true; +} + +static void mk_pci_mailbox_release(struct mk_pci_irq_vector *vector) +{ + struct mk_shared_data *shared = vector->assignment->instance->ipi_data; + struct mk_irq_mailbox *mailbox; + struct mk_irq_mailbox_entry *entry; + u64 token; + + if (!shared || vector->mailbox_slot == MK_IRQ_MAILBOX_SLOT_INVALID || + !vector->mailbox_generation) + return; + mailbox = &shared->irq_mailbox; + entry = &mailbox->entries[vector->mailbox_slot]; + for (;;) { + token = atomic64_read_acquire(&entry->pending_generation); + if (mk_irq_mailbox_generation(token) != + vector->mailbox_generation) { + atomic_inc(&mailbox->stale); + return; + } + if (atomic64_cmpxchg_release(&entry->pending_generation, + token, 0) == token) + break; + cpu_relax(); + } + mk_pci_mailbox_clear_bit(mailbox, vector->mailbox_slot); + WRITE_ONCE(entry->lifecycle_epoch, 0); + WRITE_ONCE(entry->lifecycle_generation, 0); + WRITE_ONCE(entry->device_id, 0); + WRITE_ONCE(entry->local_irq, 0); + WRITE_ONCE(entry->vector, 0); + vector->mailbox_slot = MK_IRQ_MAILBOX_SLOT_INVALID; + vector->mailbox_generation = 0; +} + +static int mk_pci_mailbox_reserve(struct mk_pci_irq_vector *vector, + unsigned int vector_index) +{ + struct mk_pci_assignment *assignment = vector->assignment; + struct mk_shared_data *shared = assignment->instance->ipi_data; + struct mk_irq_mailbox *mailbox; + struct mk_irq_mailbox_entry *entry; + u32 generation; + u32 device_id; + unsigned int slot; + u64 base; + + if (!shared) + return -ENODEV; + mailbox = &shared->irq_mailbox; + generation = (u32)atomic_inc_return(&mailbox->next_generation); + if (!generation) + generation = (u32)atomic_inc_return(&mailbox->next_generation); + if (!generation) + return -EOVERFLOW; + device_id = MK_PCI_IRQ_ID(pci_domain_nr(assignment->vf->bus), + assignment->vf->bus->number, + assignment->vf->devfn); + base = mk_irq_mailbox_token(generation, 0) | MK_IRQ_MAILBOX_MASKED; + for (slot = 0; slot < MK_IRQ_MAILBOX_SLOTS; slot++) { + entry = &mailbox->entries[slot]; + if (atomic64_read(&entry->pending_generation)) + continue; + if (atomic64_cmpxchg_release(&entry->pending_generation, 0, + base)) + continue; + WRITE_ONCE(entry->lifecycle_epoch, assignment->irq_epoch); + WRITE_ONCE(entry->lifecycle_generation, + assignment->irq_generation); + WRITE_ONCE(entry->device_id, device_id); + WRITE_ONCE(entry->local_irq, 0); + WRITE_ONCE(entry->vector, vector_index); + vector->mailbox_slot = slot; + vector->mailbox_generation = generation; + return 0; + } + return -ENOSPC; +} + +static void mk_pci_irq_retry_workfn(struct work_struct *work) +{ + struct mk_instance *instance = container_of(to_delayed_work(work), + struct mk_instance, + irq_retry_work); + struct mk_shared_data *shared = READ_ONCE(instance->ipi_data); + bool any_pending = false; + bool unmasked_pending = false; + mk_phys_cpu_t target; + unsigned int slot; + + if (!shared) + return; + for (slot = 0; slot < MK_IRQ_MAILBOX_SLOTS; slot++) { + struct mk_irq_mailbox_entry *entry; + u64 token; + + entry = &shared->irq_mailbox.entries[slot]; + token = atomic64_read_acquire(&entry->pending_generation); + + if (!mk_irq_mailbox_generation(token)) + continue; + if (mk_irq_mailbox_pending(token)) { + any_pending = true; + atomic64_or(BIT_ULL(slot & 63), + &shared->irq_mailbox.pending_bitmap[slot / 64]); + if (!mk_irq_mailbox_masked(token)) + unmasked_pending = true; + } else if (mk_irq_mailbox_consuming(token)) { + any_pending = true; + } + } + if (unmasked_pending) { + target = mk_instance_irq_route_load(instance); + if (target != MK_PHYS_CPU_INVALID) + mk_arch_send_ipi(target); + } + if (any_pending) + mod_delayed_work(system_wq, &instance->irq_retry_work, + msecs_to_jiffies(unmasked_pending ? 10 : 100)); +} + static irqreturn_t mk_pci_forward_irq(int irq, void *data) { struct mk_pci_irq_vector *vector = data; struct mk_pci_assignment *assignment = vector->assignment; - struct mk_io_irq_payload payload = { - .vector = vector - assignment->irq_vectors, - .device_id = MK_PCI_IRQ_ID(pci_domain_nr(assignment->vf->bus), - assignment->vf->bus->number, - assignment->vf->devfn), - .flags = MK_IRQ_LOW_LATENCY | MK_IRQ_EDGE_TRIGGERED, - .lifecycle_generation = READ_ONCE(assignment->irq_generation), - .lifecycle_epoch = READ_ONCE(assignment->irq_epoch), - }; - u32 local_irq = READ_ONCE(vector->local_irq); + struct mk_shared_data *shared = assignment->instance->ipi_data; + struct mk_irq_mailbox *mailbox; + struct mk_irq_mailbox_entry *entry; + mk_phys_cpu_t target; + u64 old, new; + u32 pending; + u32 local_irq; + + /* Pair with BIND's publication of the shared mailbox identity. */ + local_irq = smp_load_acquire(&vector->local_irq); if (READ_ONCE(assignment->instance->state) != MK_STATE_ACTIVE || READ_ONCE(assignment->irq_state) != MK_PCI_MSI_ACTIVE || - !local_irq) + !local_irq || !shared || + vector->mailbox_slot == MK_IRQ_MAILBOX_SLOT_INVALID || + !vector->mailbox_generation) return IRQ_HANDLED; - - payload.irq_number = local_irq; + mailbox = &shared->irq_mailbox; + entry = &mailbox->entries[vector->mailbox_slot]; + for (;;) { + old = atomic64_read_acquire(&entry->pending_generation); + if (mk_irq_mailbox_generation(old) != + vector->mailbox_generation) { + atomic_inc(&mailbox->stale); + return IRQ_HANDLED; + } + pending = mk_irq_mailbox_pending(old); + if (pending == MK_IRQ_MAILBOX_PENDING_MASK) { + atomic_inc(&mailbox->saturated); + break; + } + new = mk_irq_mailbox_token(vector->mailbox_generation, + pending + 1) | + (old & (MK_IRQ_MAILBOX_MASKED | + MK_IRQ_MAILBOX_CONSUMING)); + if (atomic64_cmpxchg(&entry->pending_generation, old, new) == old) + break; + cpu_relax(); + } + atomic64_or(BIT_ULL(vector->mailbox_slot & 63), + &mailbox->pending_bitmap[vector->mailbox_slot / 64]); + atomic_inc(&mailbox->recorded); + if (!pending) + mod_delayed_work(system_wq, &assignment->instance->irq_retry_work, + msecs_to_jiffies(10)); if (atomic64_inc_return(&vector->forwarded) == 1) pr_info("Forwarding host IRQ %u as instance IRQ %u for %s vector %u\n", irq, local_irq, pci_name(assignment->vf), - payload.vector); - if (mk_send_message_to_instance(assignment->instance, MK_MSG_IO, - MK_IO_IRQ_FORWARD, &payload, - sizeof(payload))) - pr_warn_ratelimited("Failed to forward IRQ %u for %s to instance %d\n", - irq, pci_name(assignment->vf), - assignment->instance->id); + (unsigned int)(vector - assignment->irq_vectors)); + target = mk_instance_irq_route_load(assignment->instance); + if (target != MK_PHYS_CPU_INVALID) + mk_arch_send_ipi(target); return IRQ_HANDLED; } -static unsigned int mk_pci_quiesce_irqs(struct mk_pci_assignment *assignment) +static unsigned int mk_pci_quiesce_irqs(struct mk_pci_assignment *assignment, + bool parked_force, + unsigned long deadline) { unsigned int disabled = 0; unsigned int i; + assignment->irq_mailbox_failed = false; for (i = 0; i < assignment->nr_irq_vectors; i++) WRITE_ONCE(assignment->irq_vectors[i].local_irq, 0); @@ -277,6 +496,12 @@ static unsigned int mk_pci_quiesce_irqs(struct mk_pci_assignment *assignment) disabled++; } } + for (i = 0; i < assignment->nr_irq_vectors; i++) { + struct mk_pci_irq_vector *vector = &assignment->irq_vectors[i]; + + if (!mk_pci_mailbox_quiesce(vector, parked_force, deadline)) + assignment->irq_mailbox_failed = true; + } if (assignment->irq_bound_map) bitmap_zero(assignment->irq_bound_map, assignment->nr_irq_vectors); @@ -287,16 +512,21 @@ static unsigned int mk_pci_quiesce_irqs(struct mk_pci_assignment *assignment) return disabled; } -static void mk_pci_release_irqs(struct mk_pci_assignment *assignment) +static int mk_pci_release_irqs(struct mk_pci_assignment *assignment, + bool parked_force) { + unsigned long deadline = mk_pci_mailbox_deadline(); unsigned int i; - mk_pci_quiesce_irqs(assignment); + mk_pci_quiesce_irqs(assignment, parked_force, deadline); + if (assignment->irq_mailbox_failed) + return -ETIMEDOUT; for (i = 0; i < assignment->nr_irq_vectors; i++) { struct mk_pci_irq_vector *vector = &assignment->irq_vectors[i]; if (vector->requested) free_irq(vector->host_irq, vector); + mk_pci_mailbox_release(vector); } if (assignment->nr_irq_vectors) pci_free_irq_vectors(assignment->vf); @@ -308,27 +538,37 @@ static void mk_pci_release_irqs(struct mk_pci_assignment *assignment) assignment->irq_msix = false; assignment->irq_needs_reprogram = false; assignment->irq_flr_deadline = 0; + assignment->irq_mailbox_failed = false; + return 0; } -void mk_pci_quiesce_instance_irqs(struct mk_instance *instance) +int mk_pci_quiesce_instance_irqs(struct mk_instance *instance, + bool parked_force) { struct mk_pci_assignment *assignment; + unsigned long deadline = mk_pci_mailbox_deadline(); unsigned int disabled = 0; + int ret = 0; if (!instance || instance == mk_self) - return; + return 0; lockdep_assert_held(&instance->resource_mutex); mutex_lock(&mk_pci_lease_mutex); pci_lock_rescan_remove(); list_for_each_entry(assignment, &instance->pci_assignments, - instance_node) - disabled += mk_pci_quiesce_irqs(assignment); + instance_node) { + disabled += mk_pci_quiesce_irqs(assignment, parked_force, + deadline); + if (assignment->irq_mailbox_failed) + ret = -ETIMEDOUT; + } pci_unlock_rescan_remove(); mutex_unlock(&mk_pci_lease_mutex); if (disabled) pr_info("Quiesced %u host-owned PCI IRQ vectors for instance %d\n", disabled, instance->id); + return ret; } unsigned int mk_pci_sync_instance_irq_route(struct mk_instance *instance) @@ -354,6 +594,12 @@ unsigned int mk_pci_sync_instance_irq_route(struct mk_instance *instance) } pci_unlock_rescan_remove(); mutex_unlock(&mk_pci_lease_mutex); + if (requested) { + mk_phys_cpu_t target = mk_instance_irq_route_load(instance); + + if (target != MK_PHYS_CPU_INVALID) + mk_arch_send_ipi(target); + } return requested; } @@ -373,7 +619,8 @@ static int mk_pci_setup_irqs(struct mk_pci_assignment *assignment, int nvec; int ret; - if (!request->nr_vectors) + if (!request->nr_vectors || + request->nr_vectors > MK_IRQ_MAILBOX_SLOTS) return -EINVAL; if (!request->lifecycle_generation) return -EINVAL; @@ -398,8 +645,11 @@ static int mk_pci_setup_irqs(struct mk_pci_assignment *assignment, if (!mk_pci_generation_after(request->lifecycle_generation, assignment->irq_generation)) return -ESTALE; - if (assignment->nr_irq_vectors) - mk_pci_release_irqs(assignment); + if (assignment->nr_irq_vectors) { + ret = mk_pci_release_irqs(assignment, false); + if (ret) + return ret; + } assignment->irq_epoch = request->lifecycle_epoch; assignment->irq_generation = request->lifecycle_generation; assignment->irq_state = MK_PCI_MSI_FAILED; @@ -428,6 +678,10 @@ static int mk_pci_setup_irqs(struct mk_pci_assignment *assignment, for (i = 0; i < nvec; i++) { vectors[i].assignment = assignment; vectors[i].host_irq = pci_irq_vector(assignment->vf, i); + vectors[i].mailbox_slot = MK_IRQ_MAILBOX_SLOT_INVALID; + ret = mk_pci_mailbox_reserve(&vectors[i], i); + if (ret) + goto err_release; } for (i = 0; i < nvec; i++) { ret = request_irq(vectors[i].host_irq, mk_pci_forward_irq, @@ -452,13 +706,15 @@ static int mk_pci_setup_irqs(struct mk_pci_assignment *assignment, err_release: pr_err("Failed to configure host-owned IRQ for %s vector %d: %d\n", pci_name(assignment->vf), i, ret); - mk_pci_release_irqs(assignment); + if (mk_pci_release_irqs(assignment, false)) + return -ETIMEDOUT; return ret; } static int mk_pci_bind_irq(struct mk_pci_assignment *assignment, const struct mk_pci_irq_request *request) { + struct mk_shared_data *shared = assignment->instance->ipi_data; struct mk_pci_irq_vector *vector; unsigned long delay; unsigned int count; @@ -476,6 +732,7 @@ static int mk_pci_bind_irq(struct mk_pci_assignment *assignment, (assignment->irq_state != MK_PCI_MSI_PREPARED && assignment->irq_state != MK_PCI_MSI_ACTIVE)) return -ESTALE; + if (!assignment->irq_vectors || assignment->irq_msix != request->msix) return -EINVAL; @@ -488,10 +745,24 @@ static int mk_pci_bind_irq(struct mk_pci_assignment *assignment, return -EINVAL; for (i = 0; i < count; i++) { + struct mk_irq_mailbox_entry *entry; + u64 token; + vector = &assignment->irq_vectors[request->vector + i]; local_irq = READ_ONCE(vector->local_irq); - if (!vector->requested) + if (!vector->requested || + vector->mailbox_slot == MK_IRQ_MAILBOX_SLOT_INVALID || + !vector->mailbox_generation) return -EINVAL; + entry = &shared->irq_mailbox.entries[vector->mailbox_slot]; + token = atomic64_read_acquire(&entry->pending_generation); + if (mk_irq_mailbox_generation(token) != + vector->mailbox_generation || + READ_ONCE(entry->lifecycle_epoch) != assignment->irq_epoch || + READ_ONCE(entry->lifecycle_generation) != + assignment->irq_generation || + READ_ONCE(entry->vector) != request->vector + i) + return -ESTALE; if (local_irq && local_irq != request->local_irq + i) return -EBUSY; } @@ -509,8 +780,13 @@ static int mk_pci_bind_irq(struct mk_pci_assignment *assignment, } for (i = 0; i < count; i++) { + struct mk_irq_mailbox_entry *entry; + vector = &assignment->irq_vectors[request->vector + i]; - WRITE_ONCE(vector->local_irq, request->local_irq + i); + entry = &shared->irq_mailbox.entries[vector->mailbox_slot]; + WRITE_ONCE(entry->local_irq, request->local_irq + i); + /* Make the shared identity visible before routing this vector. */ + smp_store_release(&vector->local_irq, request->local_irq + i); } bitmap_set(assignment->irq_bound_map, request->vector, count); return 0; @@ -529,7 +805,9 @@ mk_pci_restore_irqs_begin(struct mk_pci_assignment *assignment, if (assignment->irq_state != MK_PCI_MSI_ACTIVE) return -ESTALE; - mk_pci_quiesce_irqs(assignment); + mk_pci_quiesce_irqs(assignment, false, mk_pci_mailbox_deadline()); + if (assignment->irq_mailbox_failed) + return -ETIMEDOUT; assignment->irq_needs_reprogram = true; return 0; } @@ -604,8 +882,12 @@ static int mk_pci_teardown_irqs(struct mk_pci_assignment *assignment, assignment->irq_generation)) return -ESTALE; - if (assignment->nr_irq_vectors) - mk_pci_release_irqs(assignment); + if (assignment->nr_irq_vectors) { + int ret = mk_pci_release_irqs(assignment, false); + + if (ret) + return ret; + } assignment->irq_epoch = request->lifecycle_epoch; assignment->irq_generation = request->lifecycle_generation; assignment->irq_state = MK_PCI_MSI_IDLE; @@ -899,7 +1181,13 @@ static int mk_pci_reset_access(struct mk_instance *instance, /* Tombstone this serial before side effects so delayed replays reject. */ assignment->reset_generation = request->reset_generation; - mk_pci_quiesce_irqs(assignment); + mk_pci_quiesce_irqs(assignment, false, mk_pci_mailbox_deadline()); + if (assignment->irq_mailbox_failed) { + assignment->irq_state = MK_PCI_MSI_FAILED; + mk_pci_schedule_failure(assignment); + ret = -ETIMEDOUT; + goto out; + } ret = pcie_reset_flr(assignment->vf, false); if (ret) { assignment->irq_state = MK_PCI_MSI_FAILED; @@ -1038,7 +1326,7 @@ static void mk_pci_assignment_failure_work(struct work_struct *work) instance->id, ret); } - mk_instance_set_state(instance, MK_STATE_FAILED); + mk_instance_mark_failed(instance); } static void mk_pci_schedule_failure(struct mk_pci_assignment *assignment) @@ -1138,8 +1426,6 @@ static int mk_pci_iommu_validate_group(struct mk_pci_assignment *assignment) struct mk_pci_iommu_group_check check = { .vf = &assignment->vf->dev, }; - struct iommu_resv_region *region; - LIST_HEAD(resv_regions); int ret; ret = iommu_group_for_each_dev(assignment->iommu_group, &check, @@ -1158,28 +1444,65 @@ static int mk_pci_iommu_validate_group(struct mk_pci_assignment *assignment) return -EPERM; } + return 0; +} + +static int +mk_pci_iommu_validate_resv_regions(struct mk_pci_assignment *assignment) +{ + struct iommu_resv_region *region; + struct mk_memory_region *memory; + LIST_HEAD(resv_regions); + u64 memory_end, resv_end; + int ret; + ret = iommu_get_group_resv_regions(assignment->iommu_group, &resv_regions); - if (ret) { - mk_pci_iommu_free_resv_regions(&resv_regions); - return ret; - } + if (ret) + goto out; list_for_each_entry(region, &resv_regions, list) { - if (region->type != IOMMU_RESV_DIRECT && - region->type != IOMMU_RESV_DIRECT_RELAXABLE && - region->type != IOMMU_RESV_SW_MSI) + if (region->type == IOMMU_RESV_DIRECT_RELAXABLE) continue; - pr_err("IOMMU group %d for %s requires unsupported reserved region %#llx-%#llx type %u\n", - iommu_group_id(assignment->iommu_group), - pci_name(assignment->vf), - (unsigned long long)region->start, - (unsigned long long)(region->start + region->length - 1), - region->type); - ret = -EPERM; - break; + if (!region->length || + check_add_overflow((u64)region->start, + (u64)region->length - 1, &resv_end)) { + pr_err("IOMMU group %d for %s has invalid reserved region at %#llx\n", + iommu_group_id(assignment->iommu_group), + pci_name(assignment->vf), + (unsigned long long)region->start); + ret = -EOVERFLOW; + goto out; + } + + list_for_each_entry(memory, + &assignment->instance->memory_regions, list) { + resource_size_t size = resource_size(&memory->res); + + if (!size || + check_add_overflow((u64)memory->res.start, + (u64)size - 1, &memory_end)) { + ret = -EOVERFLOW; + goto out; + } + if ((u64)memory->res.start > resv_end || + memory_end < (u64)region->start) + continue; + + pr_err("Instance %d IOVA %#llx-%#llx overlaps IOMMU reserved region %#llx-%#llx type %u for %s\n", + assignment->instance->id, + (unsigned long long)memory->res.start, + (unsigned long long)memory_end, + (unsigned long long)region->start, + (unsigned long long)resv_end, region->type, + pci_name(assignment->vf)); + ret = -EPERM; + goto out; + } } + ret = 0; +out: mk_pci_iommu_free_resv_regions(&resv_regions); return ret; } @@ -1260,7 +1583,9 @@ mk_pci_quiesce_assignment(struct mk_pci_assignment *assignment) bool transactions_drained; int ret; - mk_pci_release_irqs(assignment); + ret = mk_pci_release_irqs(assignment, false); + if (ret) + return ret; if (!mk_pci_device_live(vf)) return 0; @@ -1297,7 +1622,9 @@ mk_pci_reset_assignment_for_start(struct mk_pci_assignment *assignment) struct pci_dev *vf = assignment->vf; int ret; - mk_pci_release_irqs(assignment); + ret = mk_pci_release_irqs(assignment, true); + if (ret) + return ret; assignment->irq_epoch = 0; assignment->irq_generation = 0; assignment->reset_generation = 0; @@ -1377,6 +1704,12 @@ static int mk_pci_iommu_prepare_assignment(struct mk_pci_assignment *assignment) pci_name(assignment->vf)); return -EOPNOTSUPP; } + if (!device_iommu_capable(&assignment->vf->dev, + IOMMU_CAP_CACHE_COHERENCY)) { + pr_err("Cannot assign %s without coherent IOMMU mappings\n", + pci_name(assignment->vf)); + return -EOPNOTSUPP; + } if (!assignment->instance->region_count || list_empty(&assignment->instance->memory_regions)) return -EINVAL; @@ -1386,6 +1719,9 @@ static int mk_pci_iommu_prepare_assignment(struct mk_pci_assignment *assignment) return -ENODEV; ret = mk_pci_iommu_validate_group(assignment); + if (ret) + goto err_release; + ret = mk_pci_iommu_validate_resv_regions(assignment); if (ret) goto err_release; @@ -1404,8 +1740,8 @@ static int mk_pci_iommu_prepare_assignment(struct mk_pci_assignment *assignment) if (ret) goto err_release; ret = iommu_map(assignment->iommu_domain, region->res.start, - region->res.start, size, IOMMU_READ | IOMMU_WRITE, - GFP_KERNEL); + region->res.start, size, + IOMMU_READ | IOMMU_WRITE | IOMMU_CACHE, GFP_KERNEL); if (ret) goto err_release; assignment->iommu_mapped_regions++; @@ -1785,7 +2121,8 @@ static int mk_pci_commit_assignment(struct mk_pci_assignment *assignment) return 0; } -static int mk_pci_release_assignment(struct mk_pci_assignment *assignment) +static int mk_pci_release_assignment(struct mk_pci_assignment *assignment, + struct list_head *released) { struct mk_instance *instance = assignment->instance; struct pci_dev *vf = assignment->vf; @@ -1816,9 +2153,9 @@ static int mk_pci_release_assignment(struct mk_pci_assignment *assignment) release_resources: mk_pci_iommu_release_assignment(assignment); - cancel_work_sync(&assignment->failure_work); if (assignment->inventory_moved && mk_self) { + assignment->inventory->resources_valid = false; list_move_tail(&assignment->inventory->list, &mk_self->pci_devices); instance->pci_device_count--; @@ -1827,20 +2164,32 @@ static int mk_pci_release_assignment(struct mk_pci_assignment *assignment) assignment->inventory_moved = false; } - kfree(assignment->host_driver_override); - if (assignment->host_driver && assignment->host_driver->owner) - module_put(assignment->host_driver->owner); if (!list_empty(&assignment->transaction_node)) list_del_init(&assignment->transaction_node); list_del_init(&assignment->instance_node); - pci_dev_put(assignment->pf); - pci_dev_put(vf); - kfree(assignment); + list_add_tail(&assignment->transaction_node, released); return 0; } -static int mk_pci_rollback_transaction(struct list_head *transaction) +static void mk_pci_finalize_releases(struct list_head *released) +{ + struct mk_pci_assignment *assignment, *tmp; + + list_for_each_entry_safe(assignment, tmp, released, transaction_node) { + list_del_init(&assignment->transaction_node); + cancel_work_sync(&assignment->failure_work); + kfree(assignment->host_driver_override); + if (assignment->host_driver && assignment->host_driver->owner) + module_put(assignment->host_driver->owner); + pci_dev_put(assignment->pf); + pci_dev_put(assignment->vf); + kfree(assignment); + } +} + +static int mk_pci_rollback_transaction(struct list_head *transaction, + struct list_head *released) { struct mk_pci_assignment *assignment, *tmp; int rollback_ret = 0; @@ -1848,7 +2197,7 @@ static int mk_pci_rollback_transaction(struct list_head *transaction) list_for_each_entry_safe_reverse(assignment, tmp, transaction, transaction_node) { - ret = mk_pci_release_assignment(assignment); + ret = mk_pci_release_assignment(assignment, released); if (!ret) continue; pr_crit("Failed to roll back PCI assignment for %s: %d\n", @@ -1886,6 +2235,7 @@ void mk_pci_lease_instance_init(struct mk_instance *instance) { mutex_init(&instance->resource_mutex); INIT_LIST_HEAD(&instance->pci_assignments); + INIT_DELAYED_WORK(&instance->irq_retry_work, mk_pci_irq_retry_workfn); } bool mk_pci_iommu_lease_active_locked(struct mk_instance *instance) @@ -1902,6 +2252,7 @@ int mk_pci_assign_devices(struct mk_instance *instance, int requested_count) { struct mk_pci_device *requested; + LIST_HEAD(released); LIST_HEAD(transaction); int prepared = 0; int ret = 0; @@ -1936,13 +2287,14 @@ int mk_pci_assign_devices(struct mk_instance *instance, goto out; rollback: - rollback_ret = mk_pci_rollback_transaction(&transaction); + rollback_ret = mk_pci_rollback_transaction(&transaction, &released); if (rollback_ret) ret = rollback_ret; out: pci_unlock_rescan_remove(); mutex_unlock(&mk_pci_lease_mutex); mutex_unlock(&instance->resource_mutex); + mk_pci_finalize_releases(&released); return ret; } @@ -1950,6 +2302,7 @@ int mk_pci_assign_device(struct mk_instance *instance, u16 domain, u8 bus, u8 devfn) { struct mk_pci_device *inventory; + LIST_HEAD(released); LIST_HEAD(transaction); int ret; int rollback_ret; @@ -1985,13 +2338,14 @@ int mk_pci_assign_device(struct mk_instance *instance, u16 domain, u8 bus, goto out; rollback: - rollback_ret = mk_pci_rollback_transaction(&transaction); + rollback_ret = mk_pci_rollback_transaction(&transaction, &released); if (rollback_ret) ret = rollback_ret; out: pci_unlock_rescan_remove(); mutex_unlock(&mk_pci_lease_mutex); mutex_unlock(&instance->resource_mutex); + mk_pci_finalize_releases(&released); return ret; } @@ -1999,6 +2353,7 @@ int mk_pci_unassign_device(struct mk_instance *instance, u16 domain, u8 bus, u8 devfn) { struct mk_pci_assignment *assignment; + LIST_HEAD(released); int ret; if (!instance || instance == mk_self) @@ -2019,22 +2374,26 @@ int mk_pci_unassign_device(struct mk_instance *instance, u16 domain, u8 bus, goto out; } - ret = mk_pci_release_assignment(assignment); + ret = mk_pci_release_assignment(assignment, &released); out: pci_unlock_rescan_remove(); mutex_unlock(&mk_pci_lease_mutex); mutex_unlock(&instance->resource_mutex); + mk_pci_finalize_releases(&released); return ret; } int mk_pci_release_assignments(struct mk_instance *instance) { struct mk_pci_assignment *assignment; + LIST_HEAD(released); int ret = 0; if (!instance || instance == mk_self) return 0; + mk_cpu_transaction_lock(); + down_write(&instance->control_route_sem); mutex_lock(&instance->resource_mutex); mutex_lock(&mk_pci_lease_mutex); pci_lock_rescan_remove(); @@ -2042,7 +2401,7 @@ int mk_pci_release_assignments(struct mk_instance *instance) assignment = list_last_entry(&instance->pci_assignments, struct mk_pci_assignment, instance_node); - ret = mk_pci_release_assignment(assignment); + ret = mk_pci_release_assignment(assignment, &released); if (ret) { pr_crit("Instance %d retains unsafe PCI lease for %s: %d\n", instance->id, pci_name(assignment->vf), ret); @@ -2052,6 +2411,16 @@ int mk_pci_release_assignments(struct mk_instance *instance) pci_unlock_rescan_remove(); mutex_unlock(&mk_pci_lease_mutex); mutex_unlock(&instance->resource_mutex); + up_write(&instance->control_route_sem); + mk_cpu_transaction_unlock(); + /* + * The assignment is no longer reachable by routed requests. Cancel its + * failure work after dropping the route and transaction locks because the + * worker may itself force-halt the instance and take both locks. + */ + mk_pci_finalize_releases(&released); + if (!ret) + cancel_delayed_work_sync(&instance->irq_retry_work); return ret; } @@ -2156,7 +2525,8 @@ void mk_pci_lease_system_cleanup(void) static struct mk_pci_device * mk_pci_find_assigned_bdf(u16 domain, u8 bus, u8 devfn) { - if (!mk_self || !mk_self->pci_devices_valid) + if (!mk_self || mk_self->id == 0 || + !mk_self->pci_devices_valid) return NULL; return mk_pci_find_device_bdf(&mk_self->pci_devices, From 516eddc975d954de07563da452a9ca4adcf9d6d9 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Tue, 11 Aug 2026 06:34:14 +0300 Subject: [PATCH 16/24] multikernel: expose transport reliability stats Expose a versioned per-instance snapshot for the ordered IPI ring, direct reply table, and pending IRQ mailbox. Document every counter, gauge, reset boundary, and the non-atomic modulo-u32 snapshot semantics. Signed-off-by: Nikolay Nikolaev --- Documentation/ABI/testing/multikernel | 84 ++++++++++++++ Documentation/multikernel/usage.rst | 25 ++++ include/linux/multikernel.h | 2 + kernel/multikernel/ipi.c | 1 + kernel/multikernel/kernfs.c | 160 ++++++++++++++++++++++++++ 5 files changed, 272 insertions(+) create mode 100644 Documentation/ABI/testing/multikernel diff --git a/Documentation/ABI/testing/multikernel b/Documentation/ABI/testing/multikernel new file mode 100644 index 00000000000000..74378c0dfcf4ec --- /dev/null +++ b/Documentation/ABI/testing/multikernel @@ -0,0 +1,84 @@ +What: /sys/fs/multikernel/instances//stats +Date: August 2026 +Contact: Nikolay Nikolaev +Description: + Read-only reliability snapshot for one multikernel instance. The + format is one ``key value`` pair per line. The first line is always + ``stats_version 1``. Versioned formats are append-only; readers must + ignore unknown keys. + + The snapshot is observational, not atomic with concurrent producers. + Cumulative transport counters are unsigned 32-bit event counts and + wrap modulo 2^32. They reset when the shared buffer is initialized + for a new ``spawn_epoch`` and cannot be reset through this file. + Compare deltas only between samples with the same nonzero epoch. + Gauges are unsigned point-in-time counts and have no reset operation. + + Metadata keys (class; unit; meaning): + + ``stats_version`` (metadata; version; this file format, currently 1), + ``abi_version`` (metadata; version; exact shared transport ABI), + ``snapshot_atomic`` (metadata; boolean; always 0), + ``transport_counter_bits`` (metadata; bits; always 32), + ``transport_available`` (gauge; boolean; shared buffer is mapped), and + ``spawn_epoch`` (metadata; launch identifier; changes on each start). + + ``abi_version`` identifies the exact private kernel-to-kernel transport + contract used by the running instance. Exposing it here supports + diagnostics; it is not a stable userspace ABI or a compatibility promise + between transport versions. + + Ordered IPI keys (class; unit; increment or sampled condition; reset): + + The duplex transport serializes each direction with a local spinlock and + publishes a single shared READY bit. Consequently + ``ipi.producer_contention``, ``ipi.invalid_state``, + ``ipi.cancelled_writes``, ``ipi.gate_busy``, ``ipi.slot_writing``, + ``ipi.slot_consuming``, and ``ipi.slot_cancelled`` remain zero. They are + retained in version 1 for reader compatibility. + + ``ipi.producer_contention`` (compatibility counter; always zero), + ``ipi.full_failures`` (counter; sends; either direction's head slot + prevented publication; new epoch), + ``ipi.invalid_state`` and ``ipi.cancelled_writes`` (compatibility + counters; always zero), ``ipi.gate_busy`` and ``ipi.slot_writing`` + (compatibility gauges; always zero), ``ipi.slot_ready`` (gauge; READY + slots summed across both directions; sampled), and + ``ipi.slot_consuming`` and ``ipi.slot_cancelled`` (compatibility gauges; + always zero). + + Direct reply keys (class; unit; increment or sampled condition; reset): + + ``reply.late_replies`` (counter; replies; stale exact-token claim or + completion was rejected; new epoch), ``reply.cancelled_slots`` + (counter; slots; timeout + cancelled RESERVED/WRITING; new epoch), ``reply.atomic_timeouts`` + (counter; waits; bounded atomic wait expired in cancellable + RESERVED/WRITING or indeterminate EXECUTING state; new epoch), + ``reply.indeterminate_timeouts`` (counter; waits; subset where + EXECUTING won and changed to COMMITTED; new epoch), + ``reply.occupied_failures`` (counter; + reservations; no FREE slot; new epoch), ``reply.slot_busy`` (gauge; + slots; state is not FREE; sampled), ``reply.slot_reserved`` (gauge; + slots; RESERVED; sampled), ``reply.slot_writing`` (gauge; slots; + WRITING; sampled), ``reply.slot_executing`` (gauge; slots; EXECUTING; + sampled), ``reply.slot_committed`` (gauge; slots; COMMITTED; sampled), + ``reply.slot_ready`` (gauge; slots; READY; sampled), and + ``reply.slot_abandoned`` (gauge; slots; ABANDONED; sampled). + + Pending IRQ keys (class; unit; increment or sampled condition; reset): + + ``irq.recorded`` (counter; IRQs; pending count recorded; new epoch), + ``irq.coalesced`` (counter; IRQs; recorded onto an already-pending + slot; new epoch), ``irq.masked_deferred`` (counter; IRQs; delivery + deferred while logically masked; new epoch), ``irq.stale`` (counter; + observations; epoch/generation/route validation rejected work; new + epoch), ``irq.dispatch_failed`` (counter; IRQs; validated local dispatch + failed; new epoch), ``irq.saturated`` (counter; IRQs; pending count hit + its representable limit; new epoch), ``irq.slot_active`` (gauge; slots; + nonzero generation; sampled), ``irq.slot_pending`` (gauge; slots; + nonzero pending count; sampled), ``irq.slot_masked`` (gauge; slots; + MASKED set; sampled), ``irq.slot_consuming`` (gauge; slots; CONSUMING + set; sampled), and ``irq.pending_total`` (gauge; IRQs; sum of pending + counts across active slots; sampled). +Users: multikernel management and diagnostic tools diff --git a/Documentation/multikernel/usage.rst b/Documentation/multikernel/usage.rst index bf876c209604a5..955bea57f01f64 100644 --- a/Documentation/multikernel/usage.rst +++ b/Documentation/multikernel/usage.rst @@ -89,6 +89,31 @@ Instance States - **active**: the kernel is running - **failed**: an error occurred; check ``dmesg`` +Transport Reliability Statistics +================================ + +Each instance exposes a read-only ``stats`` file. Version 1 is an +append-only key/value interface covering the duplex IPI transport, direct +reply slots, and pending IRQ mailbox. Readers must ignore unknown keys. + +The snapshot is observational rather than atomic. ``transport_available`` +reports whether the instance currently has shared transport state, and +``spawn_epoch`` identifies the launch whose counters are shown. Transport +counters are unsigned 32-bit values which may wrap and are reset when a new +epoch is initialized; compare them as modulo-32-bit deltas only between +samples with the same nonzero epoch. + +``ipi.*`` reports producer failures and current per-direction slot occupancy, +``reply.*`` reports timeout/recovery counters and reply-slot states, and +``irq.*`` reports recorded, coalesced, deferred, stale, failed, saturated, +active, pending, masked, and consuming mailbox state. Gauge values describe +only the instant at which each field was sampled. + +The duplex ready-bit protocol has no shared producer gate or intermediate +WRITING, CONSUMING, CANCELLED, or invalid states. The corresponding legacy +version-1 keys remain present and report zero; ``ipi.slot_ready`` and +``ipi.full_failures`` are summed across both link directions. + Restrictions ============ diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index c31ac674ed98cc..2da60296e91e19 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -149,6 +149,7 @@ struct mk_ipi_data { /* IPI ring buffer for queuing messages */ struct mk_ipi_ring { struct mk_ipi_data entries[MK_IPI_RING_SIZE]; /* Ring buffer entries */ + atomic_t full_failures; }; struct mk_irq_mailbox_entry { @@ -265,6 +266,7 @@ static inline void mk_ipi_ring_reset(struct mk_ipi_ring *ring) for (i = 0; i < MK_IPI_RING_SIZE; i++) WRITE_ONCE(ring->entries[i].ready, 0); + atomic_set(&ring->full_failures, 0); } static inline void mk_irq_mailbox_reset(struct mk_irq_mailbox *mailbox) diff --git a/kernel/multikernel/ipi.c b/kernel/multikernel/ipi.c index c93a9f46a5b8f8..842090d5a5470e 100644 --- a/kernel/multikernel/ipi.c +++ b/kernel/multikernel/ipi.c @@ -560,6 +560,7 @@ static int __mk_send_ipi_data(struct mk_instance *instance, slot = &endpoint->tx->entries[idx]; /* Pair with the receiver's release when it makes the slot reusable. */ if (smp_load_acquire(&slot->ready)) { + atomic_inc(&endpoint->tx->full_failures); ret = -ENOSPC; goto unlock; } diff --git a/kernel/multikernel/kernfs.c b/kernel/multikernel/kernfs.c index 90170044f31a81..a264b9ca8f95f2 100644 --- a/kernel/multikernel/kernfs.c +++ b/kernel/multikernel/kernfs.c @@ -125,6 +125,155 @@ static int status_seq_show(struct seq_file *sf, void *v) return 0; } +/* + * This is an observational snapshot, not a transaction: producers may update + * fields while the file is read. Shared transport counters are u32 values + * and restart with a new spawn_epoch, so consumers compare modulo-u32 deltas + * only between samples from the same epoch. + */ +static int stats_seq_show(struct seq_file *sf, void *v) +{ + struct kernfs_open_file *of = sf->private; + struct mk_instance *instance = of->kn->priv; + struct mk_shared_data *shared; + struct { + u64 spawn_epoch; + u64 irq_pending_total; + u32 ipi_producer_contention; + u32 ipi_full_failures; + u32 ipi_invalid_state; + u32 ipi_cancelled_writes; + u32 reply_late_replies; + u32 reply_cancelled_slots; + u32 reply_atomic_timeouts; + u32 reply_indeterminate_timeouts; + u32 reply_occupied_failures; + u32 irq_recorded; + u32 irq_coalesced; + u32 irq_masked_deferred; + u32 irq_stale; + u32 irq_dispatch_failed; + u32 irq_saturated; + u32 ipi_slots[4]; + u32 reply_states[MK_REPLY_ABANDONED + 1]; + u32 irq_active; + u32 irq_pending; + u32 irq_masked; + u32 irq_consuming; + u32 reply_busy; + u32 ipi_gate_busy; + bool available; + } stats = {}; + unsigned int i; + + down_read(&instance->control_route_sem); + shared = READ_ONCE(instance->ipi_data); + if (!shared) + goto unlock; + stats.available = true; + stats.spawn_epoch = READ_ONCE(shared->spawn_epoch); + stats.ipi_full_failures = + (u32)atomic_read(&shared->to_child.full_failures) + + (u32)atomic_read(&shared->to_parent.full_failures); + stats.reply_late_replies = atomic_read(&shared->replies.late_replies); + stats.reply_cancelled_slots = atomic_read(&shared->replies.cancelled_slots); + stats.reply_atomic_timeouts = atomic_read(&shared->replies.atomic_timeouts); + stats.reply_indeterminate_timeouts = + atomic_read(&shared->replies.indeterminate_timeouts); + stats.reply_occupied_failures = + atomic_read(&shared->replies.occupied_failures); + stats.irq_recorded = atomic_read(&shared->irq_mailbox.recorded); + stats.irq_coalesced = atomic_read(&shared->irq_mailbox.coalesced); + stats.irq_masked_deferred = + atomic_read(&shared->irq_mailbox.masked_deferred); + stats.irq_stale = atomic_read(&shared->irq_mailbox.stale); + stats.irq_dispatch_failed = + atomic_read(&shared->irq_mailbox.dispatch_failed); + stats.irq_saturated = atomic_read(&shared->irq_mailbox.saturated); + for (i = 0; i < MK_IPI_RING_SIZE; i++) { + /* Pair with the producer's release publication. */ + if (smp_load_acquire(&shared->to_child.entries[i].ready)) + stats.ipi_slots[1]++; + /* Pair with the producer's release publication. */ + if (smp_load_acquire(&shared->to_parent.entries[i].ready)) + stats.ipi_slots[1]++; + } + for (i = 0; i < MK_REPLY_SLOTS; i++) { + struct mk_reply_slot *slot = &shared->replies.slots[i]; + u64 token = atomic64_read_acquire(&slot->state_generation); + u32 state = token & (BIT(MK_REPLY_STATE_BITS) - 1); + + if (state <= MK_REPLY_ABANDONED) + stats.reply_states[state]++; + if (state != MK_REPLY_FREE) + stats.reply_busy++; + } + for (i = 0; i < MK_IRQ_MAILBOX_SLOTS; i++) { + struct mk_irq_mailbox_entry *entry = + &shared->irq_mailbox.entries[i]; + u64 token = atomic64_read_acquire(&entry->pending_generation); + + if (!mk_irq_mailbox_generation(token)) + continue; + stats.irq_active++; + if (mk_irq_mailbox_pending(token)) { + stats.irq_pending++; + stats.irq_pending_total += mk_irq_mailbox_pending(token); + } + if (mk_irq_mailbox_masked(token)) + stats.irq_masked++; + if (mk_irq_mailbox_consuming(token)) + stats.irq_consuming++; + } +unlock: + up_read(&instance->control_route_sem); + + seq_puts(sf, "stats_version 1\n"); + seq_printf(sf, "abi_version %u\n", MK_BOOT_CONTEXT_MAGIC & 0xffff); + seq_puts(sf, "snapshot_atomic 0\n"); + seq_puts(sf, "transport_counter_bits 32\n"); + seq_printf(sf, "transport_available %u\n", stats.available); + seq_printf(sf, "spawn_epoch %llu\n", + (unsigned long long)stats.spawn_epoch); +#define MK_STATS_VALUE(name, value) seq_printf(sf, name " %u\n", value) + MK_STATS_VALUE("ipi.producer_contention", stats.ipi_producer_contention); + MK_STATS_VALUE("ipi.full_failures", stats.ipi_full_failures); + MK_STATS_VALUE("ipi.invalid_state", stats.ipi_invalid_state); + MK_STATS_VALUE("ipi.cancelled_writes", stats.ipi_cancelled_writes); + MK_STATS_VALUE("ipi.gate_busy", stats.ipi_gate_busy); + MK_STATS_VALUE("ipi.slot_writing", stats.ipi_slots[0]); + MK_STATS_VALUE("ipi.slot_ready", stats.ipi_slots[1]); + MK_STATS_VALUE("ipi.slot_consuming", stats.ipi_slots[2]); + MK_STATS_VALUE("ipi.slot_cancelled", stats.ipi_slots[3]); + MK_STATS_VALUE("reply.late_replies", stats.reply_late_replies); + MK_STATS_VALUE("reply.cancelled_slots", stats.reply_cancelled_slots); + MK_STATS_VALUE("reply.atomic_timeouts", stats.reply_atomic_timeouts); + MK_STATS_VALUE("reply.indeterminate_timeouts", + stats.reply_indeterminate_timeouts); + MK_STATS_VALUE("reply.occupied_failures", stats.reply_occupied_failures); + MK_STATS_VALUE("reply.slot_busy", stats.reply_busy); + MK_STATS_VALUE("reply.slot_reserved", stats.reply_states[MK_REPLY_RESERVED]); + MK_STATS_VALUE("reply.slot_writing", stats.reply_states[MK_REPLY_WRITING]); + MK_STATS_VALUE("reply.slot_executing", stats.reply_states[MK_REPLY_EXECUTING]); + MK_STATS_VALUE("reply.slot_committed", stats.reply_states[MK_REPLY_COMMITTED]); + MK_STATS_VALUE("reply.slot_ready", stats.reply_states[MK_REPLY_READY]); + MK_STATS_VALUE("reply.slot_abandoned", stats.reply_states[MK_REPLY_ABANDONED]); + MK_STATS_VALUE("irq.recorded", stats.irq_recorded); + MK_STATS_VALUE("irq.coalesced", stats.irq_coalesced); + MK_STATS_VALUE("irq.masked_deferred", stats.irq_masked_deferred); + MK_STATS_VALUE("irq.stale", stats.irq_stale); + MK_STATS_VALUE("irq.dispatch_failed", stats.irq_dispatch_failed); + MK_STATS_VALUE("irq.saturated", stats.irq_saturated); + MK_STATS_VALUE("irq.slot_active", stats.irq_active); + MK_STATS_VALUE("irq.slot_pending", stats.irq_pending); + MK_STATS_VALUE("irq.slot_masked", stats.irq_masked); + MK_STATS_VALUE("irq.slot_consuming", stats.irq_consuming); +#undef MK_STATS_VALUE + seq_printf(sf, "irq.pending_total %llu\n", + (unsigned long long)stats.irq_pending_total); + return 0; +} + /* Root-level device_tree attribute - shows current baseline pool from kernel structures */ static int root_device_tree_seq_show(struct seq_file *sf, void *v) { @@ -342,6 +491,10 @@ static const struct kernfs_ops mk_status_ops = { .seq_show = status_seq_show, }; +static const struct kernfs_ops mk_stats_ops = { + .seq_show = stats_seq_show, +}; + /* Root-level device_tree operations */ static const struct kernfs_ops mk_root_device_tree_ops = { .seq_show = root_device_tree_seq_show, @@ -376,6 +529,13 @@ static int mk_create_instance_files(struct mk_instance *instance) pr_err("Failed to create status file for instance %s\n", instance->name); return PTR_ERR(kn); } + kn = __kernfs_create_file(instance->kn, "stats", 0444, + GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, 0, + &mk_stats_ops, instance, NULL, NULL); + if (IS_ERR(kn)) { + pr_err("Failed to create stats file for instance %s\n", instance->name); + return PTR_ERR(kn); + } kn = __kernfs_create_file(instance->kn, "device_tree", 0444, GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, 0, From 5973597d1d608280062952352e22ec9cf60df5a8 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Mon, 17 Aug 2026 22:03:32 +0000 Subject: [PATCH 17/24] multikernel: make IPI timeout recovery bounded Treat COMMITTED as a terminal timeout state during cancellation and publish replies only while the slot still holds the matching request and generation. A racing timeout can then reclaim the exact abandoned or committed slot without allowing a late reply to overwrite a reused slot. Signed-off-by: Nikolay Nikolaev --- kernel/multikernel/ipi.c | 42 ++++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/kernel/multikernel/ipi.c b/kernel/multikernel/ipi.c index 842090d5a5470e..ab9c680e574754 100644 --- a/kernel/multikernel/ipi.c +++ b/kernel/multikernel/ipi.c @@ -221,6 +221,8 @@ static bool mk_reply_cancel(struct mk_shared_data *shared, token = atomic64_read_acquire(&slot->state_generation); if (token == ready) return true; + if (token == committed) + goto timed_out; if (token == writing && atomic64_cmpxchg_release(&slot->state_generation, writing, abandoned) == writing) @@ -410,28 +412,40 @@ int mk_reply_publish_route_locked(struct mk_instance *instance, ready = mk_reply_token(reply->generation, MK_REPLY_READY); free = mk_reply_token(reply->generation, MK_REPLY_FREE); - WRITE_ONCE(slot->status, status); - WRITE_ONCE(slot->value, value); - old = atomic64_cmpxchg_release(&slot->state_generation, executing, ready); - if (old == writing) - old = atomic64_cmpxchg_release(&slot->state_generation, writing, - ready); + old = atomic64_read_acquire(&slot->state_generation); + if (READ_ONCE(slot->request_id) != reply->request_id || + READ_ONCE(slot->kind) != reply->kind) { + atomic_inc(&shared->replies.late_replies); + return -ESTALE; + } if (old == abandoned) { - atomic64_set_release(&slot->state_generation, free); + atomic64_cmpxchg_release(&slot->state_generation, abandoned, free); atomic_inc(&shared->replies.late_replies); - ret = -ESTALE; - return ret; + return -ESTALE; } if (old == committed) { - atomic64_set_release(&slot->state_generation, free); + atomic64_cmpxchg_release(&slot->state_generation, committed, free); atomic_inc(&shared->replies.late_replies); - ret = -ESTALE; - return ret; + return -ESTALE; } if (old != executing && old != writing) { atomic_inc(&shared->replies.late_replies); - ret = -EIO; - return ret; + return -EIO; + } + + WRITE_ONCE(slot->status, status); + WRITE_ONCE(slot->value, value); + if (atomic64_cmpxchg_release(&slot->state_generation, old, ready) != old) { + /* A racing timeout may abandon only this exact generation. */ + old = atomic64_read_acquire(&slot->state_generation); + if (old == abandoned) + atomic64_cmpxchg_release(&slot->state_generation, + abandoned, free); + else if (old == committed) + atomic64_cmpxchg_release(&slot->state_generation, + committed, free); + atomic_inc(&shared->replies.late_replies); + return old == abandoned || old == committed ? -ESTALE : -EIO; } target = mk_instance_irq_route_load(instance); From eb1d539581b84a1414b177b787f506b2d0a856a3 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Sun, 13 Sep 2026 07:52:39 +0000 Subject: [PATCH 18/24] pci/multikernel: fix disabled-feature builds Keep restart available when IOMMU support is disabled and provide fail-closed PCI assignment stubs when PCI support is absent. This preserves the base MultiKernel build across both supported configuration boundaries. Signed-off-by: Nikolay Nikolaev --- include/linux/multikernel.h | 7 +++++ kernel/multikernel/internal.h | 49 +++++++++++++++++++++++++++++++++++ kernel/multikernel/pci.c | 9 ++++--- 3 files changed, 61 insertions(+), 4 deletions(-) diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index 2da60296e91e19..bbc17e00762ac4 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -1307,8 +1307,15 @@ bool mk_manifest_rejected(void); /* Build the manifest for a spawn (host, kexec path) */ int mk_manifest_finalize(struct kimage *image); +#ifdef CONFIG_PCI int mk_pci_prepare_instance_start(struct mk_instance *instance); #else +static inline int mk_pci_prepare_instance_start(struct mk_instance *instance) +{ + return 0; +} +#endif +#else static inline bool multikernel_allow_emergency_restart(void) { return true; diff --git a/kernel/multikernel/internal.h b/kernel/multikernel/internal.h index 19f865c3b79068..c142cf6de5acce 100644 --- a/kernel/multikernel/internal.h +++ b/kernel/multikernel/internal.h @@ -67,6 +67,7 @@ int mk_pci_parse_bdf(const char *pci_id, int len, u16 *domain, u8 *bus, u8 *slot, u8 *func); /* pci.c */ +#ifdef CONFIG_PCI int mk_pci_lease_system_init(void); void mk_pci_lease_system_cleanup(void); void mk_pci_lease_instance_init(struct mk_instance *instance); @@ -82,6 +83,54 @@ int mk_pci_release_assignments(struct mk_instance *instance); int mk_pci_quiesce_instance_irqs(struct mk_instance *instance, bool parked_force); unsigned int mk_pci_sync_instance_irq_route(struct mk_instance *instance); +#else +static inline int mk_pci_lease_system_init(void) { return 0; } +static inline void mk_pci_lease_system_cleanup(void) { } +static inline void mk_pci_lease_instance_init(struct mk_instance *instance) { } +static inline bool +mk_pci_iommu_lease_active_locked(struct mk_instance *instance) +{ + return false; +} + +static inline int +mk_pci_assign_devices(struct mk_instance *instance, + const struct list_head *requested_devices, + int requested_count) +{ + return requested_count ? -EOPNOTSUPP : 0; +} + +static inline int +mk_pci_assign_device(struct mk_instance *instance, u16 domain, u8 bus, u8 devfn) +{ + return -EOPNOTSUPP; +} + +static inline int +mk_pci_unassign_device(struct mk_instance *instance, u16 domain, u8 bus, + u8 devfn) +{ + return -EOPNOTSUPP; +} + +static inline int mk_pci_release_assignments(struct mk_instance *instance) +{ + return 0; +} + +static inline int +mk_pci_quiesce_instance_irqs(struct mk_instance *instance, bool parked_force) +{ + return 0; +} + +static inline unsigned int +mk_pci_sync_instance_irq_route(struct mk_instance *instance) +{ + return 0; +} +#endif /* overlay.c */ extern struct kernfs_node *mk_overlay_root_kn; extern struct mutex mk_overlay_mutex; diff --git a/kernel/multikernel/pci.c b/kernel/multikernel/pci.c index 73fd839c317400..452bb5caf31fd0 100644 --- a/kernel/multikernel/pci.c +++ b/kernel/multikernel/pci.c @@ -1915,14 +1915,15 @@ static void mk_pci_iommu_system_cleanup(void) } #else static int -mk_pci_quiesce_assignment(struct mk_pci_assignment *assignment) +mk_pci_reset_assignment_for_start(struct mk_pci_assignment *assignment) { - return 0; + return -EOPNOTSUPP; } -static void -mk_pci_iommu_deactivate_assignment(struct mk_pci_assignment *assignment) +static int +mk_pci_quiesce_assignment(struct mk_pci_assignment *assignment) { + return 0; } static int mk_pci_iommu_prepare_assignment(struct mk_pci_assignment *assignment) From da2c40b01386016fc9f6ba7513dbe0ccd43691a5 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Mon, 14 Sep 2026 23:32:08 +0000 Subject: [PATCH 19/24] multikernel: make control route delivery atomic-safe Pin control-route selection through message publication and the physical doorbell with a raw spinlock. Use lifetime-stable instance pointers in atomic console and PCI paths, and keep ID lookup APIs process-context only. Drain PCI retry work before shared-ring teardown and serialize every mailbox access against publication and removal. Replace hard-IRQ mempool fallback and waiter wakeups with preallocated atomic work slots and deferred process-context wakeups. Reuse the image-owned instance during spawn and establish one instance, CPU transaction, route, and park lock order across manifest generation and pool teardown. Re-kick a full ring before reporting it through ratelimited deferred printk. Signed-off-by: Nikolay Nikolaev --- arch/x86/multikernel/pci.c | 31 +++-- arch/x86/multikernel/spawn.c | 13 +- include/linux/multikernel.h | 39 +++--- kernel/kexec_core.c | 48 +++++--- kernel/multikernel/core.c | 52 ++++---- kernel/multikernel/hotplug.c | 9 +- kernel/multikernel/internal.h | 23 ++-- kernel/multikernel/ipi.c | 94 ++++++++------ kernel/multikernel/manifest.c | 16 +-- kernel/multikernel/messaging.c | 35 ++---- kernel/multikernel/pci.c | 219 ++++++++++++++++++++++++--------- 11 files changed, 350 insertions(+), 229 deletions(-) diff --git a/arch/x86/multikernel/pci.c b/arch/x86/multikernel/pci.c index f39532f020f579..607db0c20f8075 100644 --- a/arch/x86/multikernel/pci.c +++ b/arch/x86/multikernel/pci.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -26,6 +27,9 @@ static atomic64_t mk_pci_request_id = ATOMIC64_INIT(0); static atomic64_t mk_pci_cfg_count = ATOMIC64_INIT(0); static atomic64_t mk_pci_cfg_total_ns = ATOMIC64_INIT(0); static atomic64_t mk_pci_cfg_max_ns = ATOMIC64_INIT(0); +static DEFINE_RATELIMIT_STATE(mk_pci_cfg_error_rs, + DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); #define MK_PCI_RESET_TIMEOUT_MS 70000 #ifdef CONFIG_PCI_MSI static void mk_pci_forward_irq_set_mask(struct irq_data *data, bool masked); @@ -338,7 +342,7 @@ static int mk_pci_send_irq_request(struct mk_pci_irq_request *request) return -EWOULDBLOCK; might_sleep(); if (!request->lifecycle_generation || !mk_self || - !mk_self->ipi_data) + !mk_self->ipi_data || !host_instance) return -EINVAL; request->lifecycle_epoch = READ_ONCE(mk_self->ipi_data->spawn_epoch); @@ -354,8 +358,9 @@ static int mk_pci_send_irq_request(struct mk_pci_irq_request *request) request->reply_slot = reply.slot; request->reply_generation = reply.generation; - ret = mk_send_message(host_instance->id, MK_MSG_PCI, MK_PCI_IRQ_REQUEST, - request, sizeof(*request)); + ret = mk_send_message_to_instance(host_instance, MK_MSG_PCI, + MK_PCI_IRQ_REQUEST, request, + sizeof(*request)); if (ret) { mk_reply_release(mk_self->ipi_data, &reply); return ret; @@ -702,7 +707,7 @@ int mk_pci_reset_flr(struct pci_dev *dev) if (WARN_ON_ONCE(irqs_disabled() || !in_task())) return -EWOULDBLOCK; might_sleep(); - if (!mk_self || !mk_self->ipi_data) + if (!mk_self || !mk_self->ipi_data || !host_instance) return -ENODEV; request.lifecycle_epoch = READ_ONCE(mk_self->ipi_data->spawn_epoch); @@ -723,8 +728,9 @@ int mk_pci_reset_flr(struct pci_dev *dev) request.reply_slot = reply.slot; request.reply_generation = reply.generation; - ret = mk_send_message(host_instance->id, MK_MSG_PCI, MK_PCI_RESET_REQUEST, - &request, sizeof(request)); + ret = mk_send_message_to_instance(host_instance, MK_MSG_PCI, + MK_PCI_RESET_REQUEST, &request, + sizeof(request)); if (ret) { mk_reply_release(mk_self->ipi_data, &reply); return ret; @@ -787,8 +793,9 @@ static int mk_pci_remote_config(unsigned int domain, unsigned int bus, request.reply_slot = reply.slot; request.reply_generation = reply.generation; - ret = mk_send_message(host_instance->id, MK_MSG_PCI, MK_PCI_CFG_REQUEST, - &request, sizeof(request)); + ret = mk_send_message_to_instance(host_instance, MK_MSG_PCI, + MK_PCI_CFG_REQUEST, &request, + sizeof(request)); if (ret) { mk_reply_release(mk_self->ipi_data, &reply); goto out_error; @@ -807,10 +814,10 @@ static int mk_pci_remote_config(unsigned int domain, unsigned int bus, return status; out_error: - if (ret < 0) { - pr_err_ratelimited("Multikernel PCI config request timed out or failed to send: %d\n", - ret); - } + if (ret < 0 && __ratelimit(&mk_pci_cfg_error_rs)) + printk_deferred(KERN_ERR + "Multikernel PCI config request timed out or failed to send: %d\n", + ret); return PCIBIOS_SET_FAILED; } diff --git a/arch/x86/multikernel/spawn.c b/arch/x86/multikernel/spawn.c index 78d9122761ff5b..8b51de08ad87e1 100644 --- a/arch/x86/multikernel/spawn.c +++ b/arch/x86/multikernel/spawn.c @@ -1695,7 +1695,7 @@ bool mk_pool_park_uses(phys_addr_t start, size_t size) } /** - * mk_pool_park_teardown() - Return the pool park area to the pool + * mk_arch_pool_park_teardown() - Return the pool park area to the pool * * The area is pool memory, so the chunk holding it cannot be removed * while it exists. Every parked CPU executes the park page and runs on @@ -1703,10 +1703,12 @@ bool mk_pool_park_uses(phys_addr_t start, size_t size) * context, so the area may only go back while no CPU is in the pool at * all. The next baseline builds it again, possibly elsewhere. * - * Returns 0 if there was nothing to tear down, -EBUSY if the pool still - * has CPUs. + * The generic caller holds the instance and CPU transaction locks and has + * verified that every pool CPU is back before entering this function. + * + * Returns 0 after releasing the park area or if none exists. */ -int mk_pool_park_teardown(void) +int mk_arch_pool_park_teardown(void) { if (!mk_pool) return 0; @@ -1716,9 +1718,6 @@ int mk_pool_park_teardown(void) if (!mk_pool->arch.slot) return 0; - if (!mk_pool_cpus_returned()) - return -EBUSY; - multikernel_free(mk_pool->arch.slot_phys, ALIGN(sizeof(struct mk_spawn_context), PAGE_SIZE)); mk_pool->arch.slot = NULL; diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index bbc17e00762ac4..0e29cf399a7ec0 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -356,8 +356,9 @@ void multikernel_unregister_handler(struct mk_ipi_handler *handler); * @data_size: Size of data * @type: User-defined type identifier * - * This function copies the data to per-CPU storage and sends an IPI - * to the target CPU. + * This process-context API looks up the instance by ID, copies the data to + * its endpoint, and sends an IPI to the target CPU. Atomic callers with a + * lifetime-stable instance must use multikernel_send_ipi_data_to_host(). * * Returns 0 on success, negative error code on failure */ @@ -622,17 +623,23 @@ struct mk_pending_msg; */ /** - * mk_send_message - Send a message to another CPU + * mk_send_message - Send a message to another CPU from process context * @instance_id: Target multikernel instance ID * @msg_type: Message type identifier * @subtype: Message subtype * @payload: Pointer to payload data (can be NULL) * @payload_len: Length of payload data * + * This ID-based API may sleep while looking up the target instance. Atomic + * callers with a lifetime-stable instance must use + * mk_send_message_to_instance(). + * * Returns 0 on success, negative error code on failure */ int mk_send_message(int instance_id, u32 msg_type, u32 subtype, void *payload, u32 payload_len); +int mk_send_message_to_instance(struct mk_instance *instance, u32 msg_type, + u32 subtype, void *payload, u32 payload_len); /** * mk_register_msg_handler - Register handler for specific message type @@ -748,7 +755,7 @@ size_t mk_pool_total_bytes(void); size_t mk_pool_avail_bytes(void); bool mk_pool_empty(void); int mk_pool_for_each_chunk(int (*fn)(struct mk_pool_chunk *, void *), void *data); -bool mk_pool_cpus_returned(void); +int mk_pool_park_teardown(void); /** * struct mk_pool_chunk_range - a pool chunk copied out of the chunk list @@ -774,7 +781,7 @@ int mk_pool_snapshot_chunks(struct mk_pool_chunk_range *out, int max); int mk_pool_park_setup(void); int mk_arch_pool_chunk_added(phys_addr_t start, size_t size); bool mk_pool_park_uses(phys_addr_t start, size_t size); -int mk_pool_park_teardown(void); +int mk_arch_pool_park_teardown(void); #else static inline int mk_pool_park_setup(void) { @@ -791,7 +798,7 @@ static inline bool mk_pool_park_uses(phys_addr_t start, size_t size) return false; } -static inline int mk_pool_park_teardown(void) +static inline int mk_arch_pool_park_teardown(void) { return 0; } @@ -956,6 +963,7 @@ struct mk_instance { struct mk_cpu_set *cpus; /* Set of assigned physical CPU IDs */ /* Pins the CPU selected for control messages and forwarded IRQs. */ struct rw_semaphore control_route_sem; + raw_spinlock_t control_route_lock; mk_phys_cpu_t irq_route_cpu; struct delayed_work irq_retry_work; @@ -1055,8 +1063,12 @@ mk_instance_irq_route_load(const struct mk_instance *instance) static inline void mk_instance_irq_route_store(struct mk_instance *instance, mk_phys_cpu_t target) { + unsigned long flags; + /* Publish the route after its associated control state. */ + raw_spin_lock_irqsave(&instance->control_route_lock, flags); smp_store_release(&instance->irq_route_cpu, target); + raw_spin_unlock_irqrestore(&instance->control_route_lock, flags); } /** @@ -1226,21 +1238,6 @@ void mk_instance_mem_free(struct mk_instance *instance, void *virt_addr, size_t const char *mk_state_to_string(enum mk_instance_state state); enum mk_instance_state mk_string_to_state(const char *str); -/** - * Kexec Integration Functions - * - * These functions bridge the gap between the sysfs instance management - * and the kexec multikernel system. - */ - -/** - * mk_instance_set_kexec_active() - Mark instance as active for kexec - * @mk_id: Multikernel ID from kexec system - * - * Returns 0 on success, negative error code on failure. - */ -int mk_instance_set_kexec_active(int mk_id); - /* * The declarations below are referenced from always-built code (kexec, * PCI and platform device probing, SMP setup), so they carry stubs for diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index 1907c2a8e86221..ce1422ddc5b674 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -57,6 +57,7 @@ #include #include "kexec_internal.h" +#include "multikernel/internal.h" atomic_t __kexec_lock = ATOMIC_INIT(0); @@ -608,11 +609,19 @@ void kimage_free(struct kimage *image) kimage_update_compat_pointers(NULL, KEXEC_TYPE_CRASH); if (image->type == KEXEC_TYPE_MULTIKERNEL) { + unsigned long route_flags; unsigned long i; route_instance = image->mk_instance; - if (route_instance) + if (route_instance) { + /* + * The retry worker takes the raw route lock. Drain it before + * taking the sleepable route lock to keep this ordering acyclic, + * and keep it disabled until the shared pages are gone. + */ + mk_pci_irq_retry_disable_sync(route_instance); down_write(&route_instance->control_route_sem); + } /* Stop delivery before image-owned shared pages are returned. */ #ifdef CONFIG_MULTIKERNEL @@ -636,8 +645,12 @@ void kimage_free(struct kimage *image) * instance lands in pages the allocator has already * handed to someone else. */ - image->mk_instance->ipi_data = NULL; - image->mk_instance->ipi_phys = 0; + raw_spin_lock_irqsave(&route_instance->control_route_lock, + route_flags); + route_instance->ipi_data = NULL; + route_instance->ipi_phys = 0; + raw_spin_unlock_irqrestore(&route_instance->control_route_lock, + route_flags); image->mk_instance->kimage = NULL; mk_instance_set_state(image->mk_instance, MK_STATE_READY); image->mk_instance = NULL; @@ -657,6 +670,7 @@ void kimage_free(struct kimage *image) } if (route_instance) { up_write(&route_instance->control_route_sem); + mk_pci_irq_retry_enable(route_instance); mk_instance_put(route_instance); } } @@ -1701,6 +1715,8 @@ int multikernel_kexec_by_id(int mk_id) { struct kimage *mk_image; struct mk_instance *instance; + unsigned long route_flags; + bool instance_locked = false; bool transaction_locked = false; bool route_locked = false; int cpu = -1; @@ -1723,6 +1739,8 @@ int multikernel_kexec_by_id(int mk_id) rc = -EINVAL; goto unlock; } + mutex_lock(&mk_instance_mutex); + instance_locked = true; mk_cpu_transaction_lock(); transaction_locked = true; down_write(&instance->control_route_sem); @@ -1800,6 +1818,8 @@ int multikernel_kexec_by_id(int mk_id) pr_err("Manifest finalization failed: %d\n", rc); goto unlock; } + mutex_unlock(&mk_instance_mutex); + instance_locked = false; pr_info("Manifest finalized for multikernel instance\n"); /* @@ -1811,10 +1831,14 @@ int multikernel_kexec_by_id(int mk_id) * each other's messages. */ if (mk_image->mk_ipi) { + raw_spin_lock_irqsave(&instance->control_route_lock, + route_flags); instance->ipi_phys = mk_image->mk_ipi; instance->ipi_data = phys_to_virt(mk_image->mk_ipi); instance->ipi_pages = PAGE_ALIGN(sizeof(struct mk_shared_data)) >> PAGE_SHIFT; + raw_spin_unlock_irqrestore(&instance->control_route_lock, + route_flags); } /* Reset only this parent/child link, after the old child is parked. */ @@ -1834,21 +1858,7 @@ int multikernel_kexec_by_id(int mk_id) * state before dropping the global kexec lock so another exec cannot race * this boot while the readiness handshake is pending. */ - rc = mk_instance_set_kexec_active(mk_image->mk_id); - if (rc) { - int abort_ret; - - up_write(&instance->control_route_sem); - route_locked = false; - mk_cpu_transaction_unlock(); - transaction_locked = false; - abort_ret = mk_instance_abort_spawn(instance); - - if (abort_ret) - pr_crit("Instance %d activation abort failed: %d\n", - mk_id, abort_ret); - goto unlock; - } + mk_instance_set_state(instance, MK_STATE_ACTIVE); up_write(&instance->control_route_sem); mk_cpu_transaction_unlock(); @@ -1860,6 +1870,8 @@ int multikernel_kexec_by_id(int mk_id) up_write(&instance->control_route_sem); if (transaction_locked) mk_cpu_transaction_unlock(); + if (instance_locked) + mutex_unlock(&mk_instance_mutex); kexec_unlock(); return rc; } diff --git a/kernel/multikernel/core.c b/kernel/multikernel/core.c index 74d866085431d6..4fdc579ebb08e2 100644 --- a/kernel/multikernel/core.c +++ b/kernel/multikernel/core.c @@ -320,6 +320,7 @@ struct mk_instance *mk_instance_alloc(int id, const char *name) instance->state = MK_STATE_READY; init_rwsem(&instance->control_route_sem); + raw_spin_lock_init(&instance->control_route_lock); instance->irq_route_cpu = MK_PHYS_CPU_INVALID; instance->ipi_target = MK_PHYS_CPU_INVALID; raw_spin_lock_init(&instance->ipi_endpoint.tx_lock); @@ -467,23 +468,6 @@ struct mk_instance *mk_instance_find(int mk_id) return instance; } -int mk_instance_set_kexec_active(int mk_id) -{ - struct mk_instance *instance; - - instance = mk_instance_find(mk_id); - if (!instance) { - pr_err("No sysfs instance found for multikernel ID %d\n", mk_id); - return -ENOENT; - } - - mk_instance_set_state(instance, MK_STATE_ACTIVE); - mk_instance_put(instance); - pr_info("Multikernel instance %d is now active\n", mk_id); - - return 0; -} - bool multikernel_allow_emergency_restart(void) { struct mk_instance *instance; @@ -745,7 +729,7 @@ int mk_instance_return_cpus(struct mk_instance *instance, } /** - * mk_pool_cpus_returned() - Is every pool CPU back in this kernel? + * mk_pool_cpus_returned_locked() - Is every pool CPU back in this kernel? * * True when the pool holds no free CPU and no instance owns one, so * nothing can be sitting in a park loop. Pool memory that parked CPUs @@ -754,15 +738,17 @@ int mk_instance_return_cpus(struct mk_instance *instance, * instance's set before it parks, which is why the move paths reserve * room in the destination set up front. */ -bool mk_pool_cpus_returned(void) +static bool mk_pool_cpus_returned_locked(void) { struct mk_instance *instance; bool returned = true; + lockdep_assert_held(&mk_instance_mutex); + lockdep_assert_held(&mk_cpu_transaction_mutex); + if (mk_pool && !mk_cpu_set_empty(mk_pool->cpus)) return false; - mutex_lock(&mk_instance_mutex); list_for_each_entry(instance, &mk_instance_list, list) { if (instance == mk_self) continue; @@ -773,11 +759,35 @@ bool mk_pool_cpus_returned(void) break; } } - mutex_unlock(&mk_instance_mutex); return returned; } +/** + * mk_pool_park_teardown() - Safely return the architecture park area + * + * Freeze instance membership and CPU ownership while checking that every + * CPU is home, and keep both frozen until the architecture has returned the + * executable park area to ordinary pool memory. + * + * Returns 0 on success, or -EBUSY while a pool CPU remains assigned or parked. + */ +int mk_pool_park_teardown(void) +{ + int ret; + + mutex_lock(&mk_instance_mutex); + mk_cpu_transaction_lock(); + if (!mk_pool_cpus_returned_locked()) + ret = -EBUSY; + else + ret = mk_arch_pool_park_teardown(); + mk_cpu_transaction_unlock(); + mutex_unlock(&mk_instance_mutex); + + return ret; +} + static int mk_instance_reserve_cpus(struct mk_instance *instance, const struct mk_dt_config *config) { diff --git a/kernel/multikernel/hotplug.c b/kernel/multikernel/hotplug.c index 7f529d788748ac..d9f63759fc0ca7 100644 --- a/kernel/multikernel/hotplug.c +++ b/kernel/multikernel/hotplug.c @@ -1261,7 +1261,6 @@ int mk_send_cpu_remove(int instance_id, mk_phys_cpu_t cpu_id) }; struct mk_pending_msg *pending; struct mk_instance *target_instance; - mk_phys_cpu_t route_cpu; int ret; raw_spin_lock_init(&removing.lock); @@ -1336,11 +1335,9 @@ int mk_send_cpu_remove(int instance_id, mk_phys_cpu_t cpu_id) mk_msg_pending_wait(pending, 0); goto unlock_transaction; } - route_cpu = mk_instance_irq_route_load(target_instance); - - ret = mk_send_message_to_cpu(target_instance, route_cpu, - MK_MSG_RESOURCE, MK_RES_CPU_REMOVE, - &payload, sizeof(payload)); + ret = mk_send_message_to_instance(target_instance, MK_MSG_RESOURCE, + MK_RES_CPU_REMOVE, &payload, + sizeof(payload)); if (ret < 0) { mk_msg_pending_wait(pending, 0); /* Immediate cleanup */ goto unlock_transaction; diff --git a/kernel/multikernel/internal.h b/kernel/multikernel/internal.h index c142cf6de5acce..d770ef65650814 100644 --- a/kernel/multikernel/internal.h +++ b/kernel/multikernel/internal.h @@ -23,21 +23,11 @@ int mk_instance_force_halt(struct mk_instance *instance); int mk_send_ipi_data(struct mk_instance *instance, void *data, size_t data_size, unsigned long type); struct mk_shared_data *mk_instance_halt_data(struct mk_instance *instance); -int mk_send_ipi_data_to_cpu(struct mk_instance *instance, - mk_phys_cpu_t target, void *data, - size_t data_size, unsigned long type); void mk_poll_ipi_messages(void); int mk_reply_publish_route_locked(struct mk_instance *instance, const struct mk_reply_handle *reply, s32 status, u32 value); -/* messaging.c */ -int mk_send_message_to_instance(struct mk_instance *instance, u32 msg_type, - u32 subtype, void *payload, u32 payload_len); -int mk_send_message_to_cpu(struct mk_instance *instance, - mk_phys_cpu_t target, u32 msg_type, u32 subtype, - void *payload, u32 payload_len); - /* kernfs.c */ extern struct kernfs_node *mk_root_kn; extern struct kernfs_node *mk_instances_kn; @@ -67,10 +57,12 @@ int mk_pci_parse_bdf(const char *pci_id, int len, u16 *domain, u8 *bus, u8 *slot, u8 *func); /* pci.c */ -#ifdef CONFIG_PCI +#if defined(CONFIG_MULTIKERNEL) && defined(CONFIG_PCI) int mk_pci_lease_system_init(void); void mk_pci_lease_system_cleanup(void); void mk_pci_lease_instance_init(struct mk_instance *instance); +void mk_pci_irq_retry_disable_sync(struct mk_instance *instance); +void mk_pci_irq_retry_enable(struct mk_instance *instance); bool mk_pci_iommu_lease_active_locked(struct mk_instance *instance); int mk_pci_assign_devices(struct mk_instance *instance, const struct list_head *requested_devices, @@ -87,6 +79,15 @@ unsigned int mk_pci_sync_instance_irq_route(struct mk_instance *instance); static inline int mk_pci_lease_system_init(void) { return 0; } static inline void mk_pci_lease_system_cleanup(void) { } static inline void mk_pci_lease_instance_init(struct mk_instance *instance) { } +static inline void +mk_pci_irq_retry_disable_sync(struct mk_instance *instance) +{ +} + +static inline void mk_pci_irq_retry_enable(struct mk_instance *instance) +{ +} + static inline bool mk_pci_iommu_lease_active_locked(struct mk_instance *instance) { diff --git a/kernel/multikernel/ipi.c b/kernel/multikernel/ipi.c index ab9c680e574754..480e7e4d888fa4 100644 --- a/kernel/multikernel/ipi.c +++ b/kernel/multikernel/ipi.c @@ -9,6 +9,7 @@ #include #include #include +#include #include "internal.h" static struct mk_ipi_handler *mk_handlers; @@ -19,6 +20,23 @@ static bool mk_handlers_ready; static DEFINE_RATELIMIT_STATE(mk_ipi_full_rs, DEFAULT_RATELIMIT_INTERVAL, DEFAULT_RATELIMIT_BURST); static DECLARE_WAIT_QUEUE_HEAD(mk_reply_waitq); +static void mk_reply_wake_workfn(struct work_struct *work); +static DECLARE_WORK(mk_reply_wake_work, mk_reply_wake_workfn); + +static void mk_reply_wake_workfn(struct work_struct *work) +{ + wake_up_all(&mk_reply_waitq); +} + +static void mk_reply_wake_all(void) +{ + /* + * Workqueue clears PENDING before running the callback. A false return + * therefore means a future callback is already queued; a notification + * racing with the running callback queues another execution. + */ + schedule_work(&mk_reply_wake_work); +} #define MK_REPLY_STATE_MASK (BIT(MK_REPLY_STATE_BITS) - 1) #define MK_REPLY_GENERATION_MAX (U64_MAX >> MK_REPLY_STATE_BITS) @@ -351,7 +369,7 @@ int mk_reply_claim(struct mk_instance *instance, old = atomic64_cmpxchg_release(&slot->state_generation, abandoned, free); if (old == writing || old == abandoned) - wake_up_all(&mk_reply_waitq); + mk_reply_wake_all(); atomic_inc(&shared->replies.late_replies); return -ESTALE; } @@ -395,6 +413,7 @@ int mk_reply_publish_route_locked(struct mk_instance *instance, u64 ready; u64 free; u64 old; + unsigned long flags; int ret; if (!instance || !reply || reply->slot >= MK_REPLY_SLOTS) @@ -448,13 +467,17 @@ int mk_reply_publish_route_locked(struct mk_instance *instance, return old == abandoned || old == committed ? -ESTALE : -EIO; } + raw_spin_lock_irqsave(&instance->control_route_lock, flags); target = mk_instance_irq_route_load(instance); if (target == MK_PHYS_CPU_INVALID) { ret = -ENODEV; - return ret; + goto unlock_route; } ret = 0; mk_arch_send_ipi(target); + +unlock_route: + raw_spin_unlock_irqrestore(&instance->control_route_lock, flags); return ret; } @@ -482,7 +505,7 @@ void mk_reply_scan(struct mk_shared_data *shared) u64 token = atomic64_read_acquire(&slot->state_generation); if (mk_reply_state(token) == MK_REPLY_READY) { - wake_up_all(&mk_reply_waitq); + mk_reply_wake_all(); return; } } @@ -541,32 +564,39 @@ int mk_arm_force_halt(struct mk_instance *instance) return 0; } -static int __mk_send_ipi_data(struct mk_instance *instance, - mk_phys_cpu_t target, void *data, - size_t data_size, unsigned long type) +int mk_send_ipi_data(struct mk_instance *instance, void *data, + size_t data_size, unsigned long type) { struct mk_ipi_endpoint *endpoint; struct mk_ipi_data *slot; struct mk_shared_data *shared; - unsigned long flags; + mk_phys_cpu_t target; + unsigned long route_flags; + unsigned long tx_flags; u32 idx; int ret = 0; - if (!instance || target == MK_PHYS_CPU_INVALID || - data_size > MK_MAX_DATA_SIZE || (data_size && !data)) + if (!instance || data_size > MK_MAX_DATA_SIZE || + (data_size && !data)) return -EINVAL; endpoint = &instance->ipi_endpoint; if (!READ_ONCE(endpoint->registered)) return -ESHUTDOWN; - raw_spin_lock_irqsave(&endpoint->tx_lock, flags); + raw_spin_lock_irqsave(&instance->control_route_lock, route_flags); + target = mk_instance_irq_route_load(instance); + if (target == MK_PHYS_CPU_INVALID) { + ret = -ENODEV; + goto unlock_route; + } + raw_spin_lock_irqsave(&endpoint->tx_lock, tx_flags); if (!READ_ONCE(endpoint->registered) || !endpoint->tx_enabled) { ret = -ESHUTDOWN; - goto unlock; + goto unlock_endpoint; } shared = READ_ONCE(instance->ipi_data); if (!shared) { ret = -ENODEV; - goto unlock; + goto unlock_endpoint; } if (endpoint->parent_side) WRITE_ONCE(shared->child_doorbell_cpu, target); @@ -576,7 +606,7 @@ static int __mk_send_ipi_data(struct mk_instance *instance, if (smp_load_acquire(&slot->ready)) { atomic_inc(&endpoint->tx->full_failures); ret = -ENOSPC; - goto unlock; + goto unlock_endpoint; } WRITE_ONCE(slot->sender_cpu, arch_cpu_physical_id(smp_processor_id())); WRITE_ONCE(slot->type, type); @@ -586,45 +616,29 @@ static int __mk_send_ipi_data(struct mk_instance *instance, /* Publish all message fields before the receiver observes readiness. */ smp_store_release(&slot->ready, 1); endpoint->tx_head++; -unlock: - raw_spin_unlock_irqrestore(&endpoint->tx_lock, flags); - if (!ret) + +unlock_endpoint: + raw_spin_unlock_irqrestore(&endpoint->tx_lock, tx_flags); + if (!ret || ret == -ENOSPC) mk_arch_send_ipi(target); - else if (ret == -ENOSPC && __ratelimit(&mk_ipi_full_rs)) + +unlock_route: + raw_spin_unlock_irqrestore(&instance->control_route_lock, route_flags); + if (ret == -ENOSPC && __ratelimit(&mk_ipi_full_rs)) printk_deferred(KERN_WARNING "multikernel: IPI ring full for instance %d\n", instance->id); return ret; } -int mk_send_ipi_data_to_cpu(struct mk_instance *instance, - mk_phys_cpu_t target, void *data, - size_t data_size, unsigned long type) -{ - return __mk_send_ipi_data(instance, target, data, data_size, type); -} - -int mk_send_ipi_data(struct mk_instance *instance, void *data, - size_t data_size, unsigned long type) -{ - mk_phys_cpu_t target; - int ret; - - if (!instance) - return -EINVAL; - down_read(&instance->control_route_sem); - target = mk_instance_irq_route_load(instance); - ret = __mk_send_ipi_data(instance, target, data, data_size, type); - up_read(&instance->control_route_sem); - return ret; -} - int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, unsigned long type) { - struct mk_instance *instance = mk_instance_find(instance_id); + struct mk_instance *instance; int ret; + might_sleep(); + instance = mk_instance_find(instance_id); if (!instance) return -EINVAL; ret = mk_send_ipi_data(instance, data, data_size, type); diff --git a/kernel/multikernel/manifest.c b/kernel/multikernel/manifest.c index 5ee431bf98facb..f081067b8d1af3 100644 --- a/kernel/multikernel/manifest.c +++ b/kernel/multikernel/manifest.c @@ -118,7 +118,7 @@ static int mk_manifest_collect_cpus(struct mk_instance *target, unsigned int i; int ret = 0; - mutex_lock(&mk_instance_mutex); + lockdep_assert_held(&mk_instance_mutex); list_for_each_entry(other, &mk_instance_list, list) { if (other == target) continue; @@ -137,7 +137,6 @@ static int mk_manifest_collect_cpus(struct mk_instance *target, break; } } - mutex_unlock(&mk_instance_mutex); /* * The spawn assigns logical CPU ids in this list's order, and the @@ -239,7 +238,7 @@ static int mk_manifest_add_reserved(void *fdt, struct kimage *image) if (!ret && image->mk_manifest) ret = mk_reserved_add(&r, image->mk_manifest, MK_MANIFEST_SIZE); - mutex_lock(&mk_instance_mutex); + lockdep_assert_held(&mk_instance_mutex); list_for_each_entry(other, &mk_instance_list, list) { if (ret) break; @@ -251,7 +250,6 @@ static int mk_manifest_add_reserved(void *fdt, struct kimage *image) ret = mk_reserved_add(&r, other->kimage->mk_manifest, MK_MANIFEST_SIZE); } - mutex_unlock(&mk_instance_mutex); if (!ret) { int pairs = mk_pool_park_regions(r.pair + 2 * r.n, MK_RESERVED_PAIRS - r.n); @@ -351,8 +349,12 @@ int mk_manifest_finalize(struct kimage *image) return -EINVAL; } - instance = mk_instance_find(image->mk_id); - if (!instance) { + /* + * A multikernel image owns an instance reference until kimage_free(). + * Reuse it here so callers may retain the established instance locks. + */ + instance = image->mk_instance; + if (!instance || instance->id != image->mk_id) { pr_err("Target multikernel instance %d not found\n", image->mk_id); return -ENOENT; } @@ -364,12 +366,10 @@ int mk_manifest_finalize(struct kimage *image) if (ret) { pr_err("Failed to write the boot tree for instance %d: %d\n", image->mk_id, ret); - mk_instance_put(instance); return ret == -FDT_ERR_NOSPACE ? -ENOSPC : ret; } pr_info("multikernel: boot tree for instance %d written (%u bytes)\n", image->mk_id, fdt_totalsize(fdt)); - mk_instance_put(instance); return 0; } diff --git a/kernel/multikernel/messaging.c b/kernel/multikernel/messaging.c index fbd5682d2dd7d3..1739f65aefc927 100644 --- a/kernel/multikernel/messaging.c +++ b/kernel/multikernel/messaging.c @@ -184,7 +184,8 @@ int mk_msg_pending_wait(struct mk_pending_msg *pending, unsigned long timeout_ms } /** - * mk_send_message - Send a message to another multikernel instance + * __mk_send_message - Send a message to another multikernel instance + * @instance: Lifetime-stable target instance, or NULL for ID lookup * @instance_id: Target multikernel instance ID * @msg_type: Message type identifier * @subtype: Message subtype @@ -194,7 +195,6 @@ int mk_msg_pending_wait(struct mk_pending_msg *pending, unsigned long timeout_ms * Returns 0 on success, negative error code on failure */ static int __mk_send_message(struct mk_instance *instance, int instance_id, - mk_phys_cpu_t target, u32 msg_type, u32 subtype, void *payload, u32 payload_len) { @@ -207,8 +207,9 @@ static int __mk_send_message(struct mk_instance *instance, int instance_id, /* Check if message fits in IPI buffer */ if (total_size > MK_MAX_DATA_SIZE) { - pr_err("Multikernel message too large: %zu > %d bytes\n", - total_size, MK_MAX_DATA_SIZE); + printk_deferred(KERN_ERR + "Multikernel message too large: %zu > %d bytes\n", + total_size, MK_MAX_DATA_SIZE); return -EMSGSIZE; } @@ -228,10 +229,7 @@ static int __mk_send_message(struct mk_instance *instance, int instance_id, memcpy(msg->payload, payload, payload_len); /* Send via IPI using the message type as IPI type */ - if (instance && target != MK_PHYS_CPU_INVALID) - ret = mk_send_ipi_data_to_cpu(instance, target, msg, total_size, - msg_type); - else if (instance) + if (instance) ret = mk_send_ipi_data(instance, msg, total_size, msg_type); else ret = multikernel_send_ipi_data(instance_id, msg, total_size, @@ -241,13 +239,11 @@ static int __mk_send_message(struct mk_instance *instance, int instance_id, kfree(msg); if (ret < 0) { - pr_err("Failed to send multikernel message: %d\n", ret); + printk_deferred(KERN_ERR + "Failed to send multikernel message: %d\n", ret); return ret; } - pr_debug("Multikernel message sent: type=0x%x, subtype=0x%x, len=%u to instance %d\n", - msg_type, subtype, payload_len, instance_id); - return 0; } @@ -256,24 +252,15 @@ int mk_send_message_to_instance(struct mk_instance *instance, u32 msg_type, { if (!instance) return -EINVAL; - return __mk_send_message(instance, instance->id, MK_PHYS_CPU_INVALID, + return __mk_send_message(instance, instance->id, msg_type, subtype, payload, payload_len); } -int mk_send_message_to_cpu(struct mk_instance *instance, - mk_phys_cpu_t target, u32 msg_type, u32 subtype, - void *payload, u32 payload_len) -{ - if (!instance || target == MK_PHYS_CPU_INVALID) - return -EINVAL; - return __mk_send_message(instance, instance->id, target, msg_type, - subtype, payload, payload_len); -} - int mk_send_message(int instance_id, u32 msg_type, u32 subtype, void *payload, u32 payload_len) { - return __mk_send_message(NULL, instance_id, MK_PHYS_CPU_INVALID, + might_sleep(); + return __mk_send_message(NULL, instance_id, msg_type, subtype, payload, payload_len); } EXPORT_SYMBOL(mk_send_message); diff --git a/kernel/multikernel/pci.c b/kernel/multikernel/pci.c index 452bb5caf31fd0..0b7af1f7369078 100644 --- a/kernel/multikernel/pci.c +++ b/kernel/multikernel/pci.c @@ -19,11 +19,11 @@ #include #include #include -#include #include #include #include #include +#include #include #include @@ -91,16 +91,24 @@ static DEFINE_SPINLOCK(mk_pci_active_lock); static LIST_HEAD(mk_pci_active_assignments); static bool mk_pci_notifier_registered; static bool mk_pci_control_registered; -static mempool_t *mk_pci_control_pool; +static struct mk_pci_control_work *mk_pci_control_pool; +static unsigned long *mk_pci_control_pool_busy; +static unsigned int mk_pci_control_pool_size; +static atomic_t mk_pci_control_pool_cursor = ATOMIC_INIT(0); static struct workqueue_struct *mk_pci_control_wq; -static DEFINE_SPINLOCK(mk_pci_control_lock); static DECLARE_WAIT_QUEUE_HEAD(mk_pci_control_waitq); -static unsigned int mk_pci_control_active; -static bool mk_pci_control_shutdown = true; +#define MK_PCI_CONTROL_SHUTDOWN BIT(30) +#define MK_PCI_CONTROL_ACTIVE_MASK (MK_PCI_CONTROL_SHUTDOWN - 1) +static atomic_t mk_pci_control_state = ATOMIC_INIT(MK_PCI_CONTROL_SHUTDOWN); static atomic64_t mk_pci_control_pool_exhausted = ATOMIC64_INIT(0); +static DEFINE_RATELIMIT_STATE(mk_pci_control_pool_rs, + DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); struct mk_pci_control_work { struct work_struct work; + work_func_t dispatch; + unsigned int pool_slot; s32 sender_instance_id; union { struct mk_pci_cfg_request cfg; @@ -111,52 +119,76 @@ struct mk_pci_control_work { static void mk_pci_schedule_failure(struct mk_pci_assignment *assignment); +static void mk_pci_control_wake_workfn(struct work_struct *work) +{ + wake_up_all(&mk_pci_control_waitq); +} + +static DECLARE_WORK(mk_pci_control_wake_work, + mk_pci_control_wake_workfn); + static bool mk_pci_control_handler_get(void) { - unsigned long flags; - bool acquired = false; + int state = atomic_read(&mk_pci_control_state); - spin_lock_irqsave(&mk_pci_control_lock, flags); - if (!mk_pci_control_shutdown) { - mk_pci_control_active++; - acquired = true; + for (;;) { + if (state & MK_PCI_CONTROL_SHUTDOWN) + return false; + if ((state & MK_PCI_CONTROL_ACTIVE_MASK) == + MK_PCI_CONTROL_ACTIVE_MASK) + return false; + if (atomic_try_cmpxchg(&mk_pci_control_state, &state, state + 1)) + return true; } - spin_unlock_irqrestore(&mk_pci_control_lock, flags); - return acquired; } static void mk_pci_control_handler_put(void) { - unsigned long flags; - bool drained; + int state = atomic_read(&mk_pci_control_state); + int new_state; - spin_lock_irqsave(&mk_pci_control_lock, flags); - WARN_ON_ONCE(!mk_pci_control_active); - if (mk_pci_control_active) - mk_pci_control_active--; - drained = !mk_pci_control_active; - spin_unlock_irqrestore(&mk_pci_control_lock, flags); - if (drained) - wake_up_all(&mk_pci_control_waitq); + for (;;) { + if (!(state & MK_PCI_CONTROL_ACTIVE_MASK)) + return; + new_state = state - 1; + if (atomic_try_cmpxchg(&mk_pci_control_state, &state, new_state)) + break; + } + if (new_state == MK_PCI_CONTROL_SHUTDOWN) + schedule_work(&mk_pci_control_wake_work); } static void mk_pci_control_shutdown_begin(void) { - unsigned long flags; - - spin_lock_irqsave(&mk_pci_control_lock, flags); - mk_pci_control_shutdown = true; - spin_unlock_irqrestore(&mk_pci_control_lock, flags); + atomic_or(MK_PCI_CONTROL_SHUTDOWN, &mk_pci_control_state); } static void mk_pci_control_shutdown_end(void) { - unsigned long flags; + WARN_ON_ONCE(atomic_read(&mk_pci_control_state) != + MK_PCI_CONTROL_SHUTDOWN); + atomic_set(&mk_pci_control_state, 0); +} - spin_lock_irqsave(&mk_pci_control_lock, flags); - mk_pci_control_active = 0; - mk_pci_control_shutdown = false; - spin_unlock_irqrestore(&mk_pci_control_lock, flags); +static struct mk_pci_control_work *mk_pci_control_work_get(void) +{ + unsigned int start; + unsigned int slot; + unsigned int i; + + start = (unsigned int)atomic_inc_return(&mk_pci_control_pool_cursor); + for (i = 0; i < mk_pci_control_pool_size; i++) { + slot = (start + i) % mk_pci_control_pool_size; + if (!test_and_set_bit_lock(slot, mk_pci_control_pool_busy)) + return &mk_pci_control_pool[slot]; + } + return NULL; +} + +static void +mk_pci_control_work_put(struct mk_pci_control_work *control_work) +{ + clear_bit_unlock(control_work->pool_slot, mk_pci_control_pool_busy); } static bool mk_pci_device_live(struct pci_dev *pdev) @@ -380,14 +412,17 @@ static void mk_pci_irq_retry_workfn(struct work_struct *work) struct mk_instance *instance = container_of(to_delayed_work(work), struct mk_instance, irq_retry_work); - struct mk_shared_data *shared = READ_ONCE(instance->ipi_data); + struct mk_shared_data *shared; bool any_pending = false; bool unmasked_pending = false; mk_phys_cpu_t target; + unsigned long flags; unsigned int slot; + raw_spin_lock_irqsave(&instance->control_route_lock, flags); + shared = READ_ONCE(instance->ipi_data); if (!shared) - return; + goto unlock; for (slot = 0; slot < MK_IRQ_MAILBOX_SLOTS; slot++) { struct mk_irq_mailbox_entry *entry; u64 token; @@ -412,6 +447,8 @@ static void mk_pci_irq_retry_workfn(struct work_struct *work) if (target != MK_PHYS_CPU_INVALID) mk_arch_send_ipi(target); } +unlock: + raw_spin_unlock_irqrestore(&instance->control_route_lock, flags); if (any_pending) mod_delayed_work(system_wq, &instance->irq_retry_work, msecs_to_jiffies(unmasked_pending ? 10 : 100)); @@ -421,23 +458,29 @@ static irqreturn_t mk_pci_forward_irq(int irq, void *data) { struct mk_pci_irq_vector *vector = data; struct mk_pci_assignment *assignment = vector->assignment; - struct mk_shared_data *shared = assignment->instance->ipi_data; + struct mk_instance *instance = assignment->instance; + struct mk_shared_data *shared; struct mk_irq_mailbox *mailbox; struct mk_irq_mailbox_entry *entry; + bool first_forward = false; + bool schedule_retry = false; mk_phys_cpu_t target; u64 old, new; u32 pending; u32 local_irq; + unsigned long flags; /* Pair with BIND's publication of the shared mailbox identity. */ local_irq = smp_load_acquire(&vector->local_irq); - if (READ_ONCE(assignment->instance->state) != MK_STATE_ACTIVE || + raw_spin_lock_irqsave(&instance->control_route_lock, flags); + shared = READ_ONCE(instance->ipi_data); + if (READ_ONCE(instance->state) != MK_STATE_ACTIVE || READ_ONCE(assignment->irq_state) != MK_PCI_MSI_ACTIVE || !local_irq || !shared || vector->mailbox_slot == MK_IRQ_MAILBOX_SLOT_INVALID || !vector->mailbox_generation) - return IRQ_HANDLED; + goto unlock; mailbox = &shared->irq_mailbox; entry = &mailbox->entries[vector->mailbox_slot]; for (;;) { @@ -445,7 +488,7 @@ static irqreturn_t mk_pci_forward_irq(int irq, void *data) if (mk_irq_mailbox_generation(old) != vector->mailbox_generation) { atomic_inc(&mailbox->stale); - return IRQ_HANDLED; + goto unlock; } pending = mk_irq_mailbox_pending(old); if (pending == MK_IRQ_MAILBOX_PENDING_MASK) { @@ -464,15 +507,21 @@ static irqreturn_t mk_pci_forward_irq(int irq, void *data) &mailbox->pending_bitmap[vector->mailbox_slot / 64]); atomic_inc(&mailbox->recorded); if (!pending) - mod_delayed_work(system_wq, &assignment->instance->irq_retry_work, - msecs_to_jiffies(10)); - if (atomic64_inc_return(&vector->forwarded) == 1) - pr_info("Forwarding host IRQ %u as instance IRQ %u for %s vector %u\n", - irq, local_irq, pci_name(assignment->vf), - (unsigned int)(vector - assignment->irq_vectors)); - target = mk_instance_irq_route_load(assignment->instance); + schedule_retry = true; + first_forward = atomic64_inc_return(&vector->forwarded) == 1; + target = mk_instance_irq_route_load(instance); if (target != MK_PHYS_CPU_INVALID) mk_arch_send_ipi(target); +unlock: + raw_spin_unlock_irqrestore(&instance->control_route_lock, flags); + if (schedule_retry) + mod_delayed_work(system_wq, &instance->irq_retry_work, + msecs_to_jiffies(10)); + if (first_forward) + printk_deferred(KERN_INFO + "Forwarding host IRQ %u as instance IRQ %u for %s vector %u\n", + irq, local_irq, pci_name(assignment->vf), + (unsigned int)(vector - assignment->irq_vectors)); return IRQ_HANDLED; } @@ -595,10 +644,14 @@ unsigned int mk_pci_sync_instance_irq_route(struct mk_instance *instance) pci_unlock_rescan_remove(); mutex_unlock(&mk_pci_lease_mutex); if (requested) { - mk_phys_cpu_t target = mk_instance_irq_route_load(instance); + mk_phys_cpu_t target; + unsigned long flags; + raw_spin_lock_irqsave(&instance->control_route_lock, flags); + target = mk_instance_irq_route_load(instance); if (target != MK_PHYS_CPU_INVALID) mk_arch_send_ipi(target); + raw_spin_unlock_irqrestore(&instance->control_route_lock, flags); } return requested; } @@ -1038,7 +1091,7 @@ static void mk_pci_cfg_work_fn(struct work_struct *work) up_read(&instance->control_route_sem); mk_instance_put(instance); out: - mempool_free(control_work, mk_pci_control_pool); + return; } static int mk_pci_irq_access(struct mk_instance *instance, @@ -1142,7 +1195,7 @@ static void mk_pci_irq_work_fn(struct work_struct *work) up_read(&instance->control_route_sem); mk_instance_put(instance); out: - mempool_free(control_work, mk_pci_control_pool); + return; } static int mk_pci_reset_access(struct mk_instance *instance, @@ -1242,7 +1295,16 @@ static void mk_pci_reset_work_fn(struct work_struct *work) up_read(&instance->control_route_sem); mk_instance_put(instance); out: - mempool_free(control_work, mk_pci_control_pool); + return; +} + +static void mk_pci_control_work_fn(struct work_struct *work) +{ + struct mk_pci_control_work *control_work = + container_of(work, struct mk_pci_control_work, work); + + control_work->dispatch(work); + mk_pci_control_work_put(control_work); } static void mk_pci_control_msg_handler(u32 msg_type, u32 subtype, @@ -1283,13 +1345,15 @@ static void mk_pci_control_msg_handler(u32 msg_type, u32 subtype, * keeps the hardirq receive path allocation-safe without changing reply * or route validation. Duplicate traffic is outside the cooperative ABI. */ - control_work = mempool_alloc(mk_pci_control_pool, GFP_ATOMIC); + control_work = mk_pci_control_work_get(); if (!control_work) { atomic64_inc(&mk_pci_control_pool_exhausted); - pr_warn_ratelimited("Multikernel PCI control work pool exhausted\n"); + if (__ratelimit(&mk_pci_control_pool_rs)) + printk_deferred(KERN_WARNING + "Multikernel PCI control work pool exhausted\n"); goto out; } - INIT_WORK(&control_work->work, work_fn); + control_work->dispatch = work_fn; memcpy(&control_work->request, payload, request_size); control_work->sender_instance_id = sender_instance_id; switch (subtype) { @@ -1304,7 +1368,7 @@ static void mk_pci_control_msg_handler(u32 msg_type, u32 subtype, break; } if (!queue_work(mk_pci_control_wq, &control_work->work)) - mempool_free(control_work, mk_pci_control_pool); + mk_pci_control_work_put(control_work); out: mk_pci_control_handler_put(); } @@ -2239,6 +2303,16 @@ void mk_pci_lease_instance_init(struct mk_instance *instance) INIT_DELAYED_WORK(&instance->irq_retry_work, mk_pci_irq_retry_workfn); } +void mk_pci_irq_retry_disable_sync(struct mk_instance *instance) +{ + disable_delayed_work_sync(&instance->irq_retry_work); +} + +void mk_pci_irq_retry_enable(struct mk_instance *instance) +{ + enable_delayed_work(&instance->irq_retry_work); +} + bool mk_pci_iommu_lease_active_locked(struct mk_instance *instance) { if (!instance) @@ -2450,8 +2524,8 @@ int mk_pci_prepare_instance_start(struct mk_instance *instance) int mk_pci_lease_system_init(void) { + unsigned int i; unsigned int pool_size; - size_t work_size = sizeof(struct mk_pci_control_work); int ret; ret = mk_pci_iommu_system_init(); @@ -2466,17 +2540,31 @@ int mk_pci_lease_system_init(void) if (mk_self && mk_self->id == 0) { pool_size = max_t(unsigned int, num_possible_cpus(), 1) * MK_REPLY_SLOTS; - mk_pci_control_pool = mempool_create_kmalloc_pool(pool_size, work_size); + mk_pci_control_pool = kcalloc(pool_size, + sizeof(*mk_pci_control_pool), + GFP_KERNEL); if (!mk_pci_control_pool) { ret = -ENOMEM; goto unregister_notifier; } + mk_pci_control_pool_busy = bitmap_zalloc(pool_size, GFP_KERNEL); + if (!mk_pci_control_pool_busy) { + ret = -ENOMEM; + goto free_pool; + } + mk_pci_control_pool_size = pool_size; + atomic_set(&mk_pci_control_pool_cursor, 0); + for (i = 0; i < pool_size; i++) { + INIT_WORK(&mk_pci_control_pool[i].work, + mk_pci_control_work_fn); + mk_pci_control_pool[i].pool_slot = i; + } mk_pci_control_wq = alloc_workqueue("mk-pci-control", WQ_UNBOUND | WQ_MEM_RECLAIM, 0); if (!mk_pci_control_wq) { ret = -ENOMEM; - goto destroy_pool; + goto free_pool_bitmap; } mk_pci_control_shutdown_end(); ret = mk_register_msg_handler(MK_MSG_PCI, @@ -2491,8 +2579,12 @@ int mk_pci_lease_system_init(void) mk_pci_control_shutdown_begin(); destroy_workqueue(mk_pci_control_wq); mk_pci_control_wq = NULL; -destroy_pool: - mempool_destroy(mk_pci_control_pool); +free_pool_bitmap: + bitmap_free(mk_pci_control_pool_busy); + mk_pci_control_pool_busy = NULL; + mk_pci_control_pool_size = 0; +free_pool: + kfree(mk_pci_control_pool); mk_pci_control_pool = NULL; unregister_notifier: bus_unregister_notifier(&pci_bus_type, &mk_pci_bus_notifier); @@ -2508,13 +2600,18 @@ void mk_pci_lease_system_cleanup(void) mk_unregister_msg_handler(MK_MSG_PCI, mk_pci_control_msg_handler); mk_pci_control_registered = false; - wait_event(mk_pci_control_waitq, !READ_ONCE(mk_pci_control_active)); + wait_event(mk_pci_control_waitq, + !(atomic_read(&mk_pci_control_state) & + MK_PCI_CONTROL_ACTIVE_MASK)); } if (mk_pci_control_wq) { destroy_workqueue(mk_pci_control_wq); mk_pci_control_wq = NULL; } - mempool_destroy(mk_pci_control_pool); + bitmap_free(mk_pci_control_pool_busy); + mk_pci_control_pool_busy = NULL; + mk_pci_control_pool_size = 0; + kfree(mk_pci_control_pool); mk_pci_control_pool = NULL; if (mk_pci_notifier_registered) { bus_unregister_notifier(&pci_bus_type, &mk_pci_bus_notifier); From 9f303cc952b55d9291491a253fc5757634f1679c Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Mon, 14 Sep 2026 23:51:00 +0000 Subject: [PATCH 20/24] tty: make multikernel host receive locks IRQ-safe Signed-off-by: Nikolay Nikolaev --- drivers/tty/mktty.c | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/drivers/tty/mktty.c b/drivers/tty/mktty.c index 535c6c1fe8a3e7..00350a6182be65 100644 --- a/drivers/tty/mktty.c +++ b/drivers/tty/mktty.c @@ -54,14 +54,14 @@ struct mktty_host_conn { int instance_id; /* -1 = not connected */ struct list_head list; wait_queue_head_t wait; - spinlock_t rx_lock; + raw_spinlock_t rx_lock; char *rx_buf; int rx_head; int rx_tail; }; static LIST_HEAD(mktty_host_conns); -static DEFINE_SPINLOCK(mktty_host_conns_lock); +static DEFINE_RAW_SPINLOCK(mktty_host_conns_lock); static struct mk_ipi_handler *mktty_host_handler; static struct mktty_host_conn *mktty_find_conn(int instance_id) @@ -78,6 +78,7 @@ static struct mktty_host_conn *mktty_find_conn(int instance_id) static int mktty_host_open(struct inode *inode, struct file *filp) { struct mktty_host_conn *conn; + unsigned long flags; conn = kzalloc(sizeof(*conn), GFP_KERNEL); if (!conn) @@ -91,13 +92,13 @@ static int mktty_host_open(struct inode *inode, struct file *filp) conn->instance_id = -1; init_waitqueue_head(&conn->wait); - spin_lock_init(&conn->rx_lock); + raw_spin_lock_init(&conn->rx_lock); conn->rx_head = 0; conn->rx_tail = 0; - spin_lock(&mktty_host_conns_lock); + raw_spin_lock_irqsave(&mktty_host_conns_lock, flags); list_add(&conn->list, &mktty_host_conns); - spin_unlock(&mktty_host_conns_lock); + raw_spin_unlock_irqrestore(&mktty_host_conns_lock, flags); filp->private_data = conn; return 0; @@ -106,10 +107,11 @@ static int mktty_host_open(struct inode *inode, struct file *filp) static int mktty_host_release(struct inode *inode, struct file *filp) { struct mktty_host_conn *conn = filp->private_data; + unsigned long flags; - spin_lock(&mktty_host_conns_lock); + raw_spin_lock_irqsave(&mktty_host_conns_lock, flags); list_del(&conn->list); - spin_unlock(&mktty_host_conns_lock); + raw_spin_unlock_irqrestore(&mktty_host_conns_lock, flags); kfree(conn->rx_buf); kfree(conn); @@ -128,22 +130,22 @@ static ssize_t mktty_host_read(struct file *filp, char __user *buf, return -ENOTCONN; while (copied == 0) { - spin_lock_irqsave(&conn->rx_lock, flags); + raw_spin_lock_irqsave(&conn->rx_lock, flags); while (copied < count && conn->rx_head != conn->rx_tail) { char c = conn->rx_buf[conn->rx_tail]; conn->rx_tail = (conn->rx_tail + 1) % MKTTY_RX_BUF_SIZE; - spin_unlock_irqrestore(&conn->rx_lock, flags); + raw_spin_unlock_irqrestore(&conn->rx_lock, flags); if (put_user(c, buf + copied)) { return copied > 0 ? copied : -EFAULT; } copied++; - spin_lock_irqsave(&conn->rx_lock, flags); + raw_spin_lock_irqsave(&conn->rx_lock, flags); } - spin_unlock_irqrestore(&conn->rx_lock, flags); + raw_spin_unlock_irqrestore(&conn->rx_lock, flags); if (copied > 0) break; @@ -240,10 +242,10 @@ static __poll_t mktty_host_poll(struct file *filp, poll_table *wait) poll_wait(filp, &conn->wait, wait); - spin_lock_irqsave(&conn->rx_lock, flags); + raw_spin_lock_irqsave(&conn->rx_lock, flags); if (conn->rx_head != conn->rx_tail) mask |= EPOLLIN | EPOLLRDNORM; - spin_unlock_irqrestore(&conn->rx_lock, flags); + raw_spin_unlock_irqrestore(&conn->rx_lock, flags); mask |= EPOLLOUT | EPOLLWRNORM; return mask; @@ -278,10 +280,10 @@ static void mktty_host_ipi_handler(struct mk_ipi_data *data, void *ctx) if (msg->len > MKTTY_MAX_DATA) return; - spin_lock_irqsave(&mktty_host_conns_lock, flags); + raw_spin_lock_irqsave(&mktty_host_conns_lock, flags); conn = mktty_find_conn(msg->console_id); if (conn) { - spin_lock(&conn->rx_lock); + raw_spin_lock(&conn->rx_lock); for (i = 0; i < msg->len; i++) { space = (conn->rx_tail - conn->rx_head - 1 + MKTTY_RX_BUF_SIZE) % MKTTY_RX_BUF_SIZE; if (space == 0) @@ -289,10 +291,10 @@ static void mktty_host_ipi_handler(struct mk_ipi_data *data, void *ctx) conn->rx_buf[conn->rx_head] = msg->data[i]; conn->rx_head = (conn->rx_head + 1) % MKTTY_RX_BUF_SIZE; } - spin_unlock(&conn->rx_lock); + raw_spin_unlock(&conn->rx_lock); wake_up_interruptible(&conn->wait); } - spin_unlock_irqrestore(&mktty_host_conns_lock, flags); + raw_spin_unlock_irqrestore(&mktty_host_conns_lock, flags); } static int mktty_host_init(void) From 7338170518a97a16894b43aac87ff0c2c5663f75 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Tue, 15 Sep 2026 00:05:58 +0000 Subject: [PATCH 21/24] multikernel: initialize resource state without PCI Signed-off-by: Nikolay Nikolaev --- kernel/multikernel/core.c | 2 ++ kernel/multikernel/pci.c | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/multikernel/core.c b/kernel/multikernel/core.c index 4fdc579ebb08e2..6653e212589ec5 100644 --- a/kernel/multikernel/core.c +++ b/kernel/multikernel/core.c @@ -329,6 +329,8 @@ struct mk_instance *mk_instance_alloc(int id, const char *name) INIT_LIST_HEAD(&instance->memory_regions); INIT_LIST_HEAD(&instance->list); INIT_LIST_HEAD(&instance->pci_devices); + mutex_init(&instance->resource_mutex); + INIT_LIST_HEAD(&instance->pci_assignments); INIT_LIST_HEAD(&instance->platform_devices); mk_pci_lease_instance_init(instance); kref_init(&instance->refcount); diff --git a/kernel/multikernel/pci.c b/kernel/multikernel/pci.c index 0b7af1f7369078..7a4f2a7c173556 100644 --- a/kernel/multikernel/pci.c +++ b/kernel/multikernel/pci.c @@ -2298,8 +2298,6 @@ static int mk_pci_commit_transaction(struct list_head *transaction) void mk_pci_lease_instance_init(struct mk_instance *instance) { - mutex_init(&instance->resource_mutex); - INIT_LIST_HEAD(&instance->pci_assignments); INIT_DELAYED_WORK(&instance->irq_retry_work, mk_pci_irq_retry_workfn); } From 61865d2dda84c73735fd483b0c52a3a76ec86b5c Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Tue, 15 Sep 2026 01:12:24 +0000 Subject: [PATCH 22/24] tty: defer multikernel host wakeups from hard IRQ Signed-off-by: Nikolay Nikolaev --- drivers/tty/mktty.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/drivers/tty/mktty.c b/drivers/tty/mktty.c index 00350a6182be65..ae8ebfde0178bc 100644 --- a/drivers/tty/mktty.c +++ b/drivers/tty/mktty.c @@ -27,6 +27,7 @@ #include #include #include +#include #define MKTTY_IPI_TYPE 0x4D4B5459U /* "MKTY" */ #define MKTTY_MAX_DATA (MK_MAX_DATA_SIZE - 16) @@ -54,6 +55,7 @@ struct mktty_host_conn { int instance_id; /* -1 = not connected */ struct list_head list; wait_queue_head_t wait; + struct work_struct wake_work; raw_spinlock_t rx_lock; char *rx_buf; int rx_head; @@ -64,6 +66,14 @@ static LIST_HEAD(mktty_host_conns); static DEFINE_RAW_SPINLOCK(mktty_host_conns_lock); static struct mk_ipi_handler *mktty_host_handler; +static void mktty_host_wake_workfn(struct work_struct *work) +{ + struct mktty_host_conn *conn = + container_of(work, struct mktty_host_conn, wake_work); + + wake_up_interruptible(&conn->wait); +} + static struct mktty_host_conn *mktty_find_conn(int instance_id) { struct mktty_host_conn *conn; @@ -92,6 +102,7 @@ static int mktty_host_open(struct inode *inode, struct file *filp) conn->instance_id = -1; init_waitqueue_head(&conn->wait); + INIT_WORK(&conn->wake_work, mktty_host_wake_workfn); raw_spin_lock_init(&conn->rx_lock); conn->rx_head = 0; conn->rx_tail = 0; @@ -113,6 +124,7 @@ static int mktty_host_release(struct inode *inode, struct file *filp) list_del(&conn->list); raw_spin_unlock_irqrestore(&mktty_host_conns_lock, flags); + cancel_work_sync(&conn->wake_work); kfree(conn->rx_buf); kfree(conn); return 0; @@ -292,7 +304,7 @@ static void mktty_host_ipi_handler(struct mk_ipi_data *data, void *ctx) conn->rx_head = (conn->rx_head + 1) % MKTTY_RX_BUF_SIZE; } raw_spin_unlock(&conn->rx_lock); - wake_up_interruptible(&conn->wait); + schedule_work(&conn->wake_work); } raw_spin_unlock_irqrestore(&mktty_host_conns_lock, flags); } From 6b47d14377678a68094c715e5a9d2337d10d3255 Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Tue, 15 Sep 2026 01:24:40 +0000 Subject: [PATCH 23/24] multikernel: defer message completions from hard IRQ Signed-off-by: Nikolay Nikolaev --- kernel/multikernel/messaging.c | 36 ++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/kernel/multikernel/messaging.c b/kernel/multikernel/messaging.c index 1739f65aefc927..c56434c9740084 100644 --- a/kernel/multikernel/messaging.c +++ b/kernel/multikernel/messaging.c @@ -11,6 +11,7 @@ #include #include #include +#include #include "internal.h" /* Pending message tracking for request-response pattern */ @@ -20,11 +21,12 @@ struct mk_pending_msg { u64 resource_id; /* Resource identifier (physical CPU ID, PFN, etc.) */ int result; /* Operation result */ struct completion done; /* Completion for waiting */ + struct work_struct complete_work; struct list_head list; /* List linkage */ }; static LIST_HEAD(mk_pending_msgs); -static DEFINE_SPINLOCK(mk_pending_msgs_lock); +static DEFINE_RAW_SPINLOCK(mk_pending_msgs_lock); /* Per-type message handler registry */ struct mk_msg_type_handler { @@ -96,6 +98,14 @@ static void mk_message_type_ipi_callback(struct mk_ipi_data *data, void *ctx) * Pending message tracking for request-response pattern */ +static void mk_msg_pending_complete_workfn(struct work_struct *work) +{ + struct mk_pending_msg *pending = + container_of(work, struct mk_pending_msg, complete_work); + + complete(&pending->done); +} + /** * mk_msg_pending_add - Register a pending operation awaiting response * @msg_type: Message type @@ -118,10 +128,11 @@ struct mk_pending_msg *mk_msg_pending_add(u32 msg_type, u32 operation, u64 resou pending->resource_id = resource_id; pending->result = -ETIMEDOUT; /* Default to timeout */ init_completion(&pending->done); + INIT_WORK(&pending->complete_work, mk_msg_pending_complete_workfn); - spin_lock_irqsave(&mk_pending_msgs_lock, flags); + raw_spin_lock_irqsave(&mk_pending_msgs_lock, flags); list_add(&pending->list, &mk_pending_msgs); - spin_unlock_irqrestore(&mk_pending_msgs_lock, flags); + raw_spin_unlock_irqrestore(&mk_pending_msgs_lock, flags); return pending; } @@ -140,17 +151,17 @@ void mk_msg_pending_complete(u32 msg_type, u32 operation, u64 resource_id, int r struct mk_pending_msg *pending; unsigned long flags; - spin_lock_irqsave(&mk_pending_msgs_lock, flags); + raw_spin_lock_irqsave(&mk_pending_msgs_lock, flags); list_for_each_entry(pending, &mk_pending_msgs, list) { if (pending->msg_type == msg_type && pending->operation == operation && pending->resource_id == resource_id) { pending->result = result; - complete(&pending->done); + schedule_work(&pending->complete_work); break; } } - spin_unlock_irqrestore(&mk_pending_msgs_lock, flags); + raw_spin_unlock_irqrestore(&mk_pending_msgs_lock, flags); } /** @@ -164,20 +175,21 @@ int mk_msg_pending_wait(struct mk_pending_msg *pending, unsigned long timeout_ms { unsigned long timeout = msecs_to_jiffies(timeout_ms); unsigned long flags; + bool timed_out; int result; - if (!wait_for_completion_timeout(&pending->done, timeout)) { + timed_out = !wait_for_completion_timeout(&pending->done, timeout); + if (timed_out) { pr_err("Timeout waiting for operation 0x%x on resource %llu\n", pending->operation, pending->resource_id); - result = -ETIMEDOUT; - } else { - result = pending->result; } /* Remove from list and free */ - spin_lock_irqsave(&mk_pending_msgs_lock, flags); + raw_spin_lock_irqsave(&mk_pending_msgs_lock, flags); list_del(&pending->list); - spin_unlock_irqrestore(&mk_pending_msgs_lock, flags); + result = timed_out ? -ETIMEDOUT : pending->result; + raw_spin_unlock_irqrestore(&mk_pending_msgs_lock, flags); + cancel_work_sync(&pending->complete_work); kfree(pending); return result; From d0dd880b500e3aff03ac7e069a004f33be0cb95b Mon Sep 17 00:00:00 2001 From: Nikolay Nikolaev Date: Tue, 15 Sep 2026 05:18:50 +0000 Subject: [PATCH 24/24] multikernel: preallocate IPI control work Signed-off-by: Nikolay Nikolaev --- kernel/multikernel/core.c | 150 ++++++++++-- kernel/multikernel/hotplug.c | 436 +++++++++++++++++++++------------- kernel/multikernel/internal.h | 3 +- 3 files changed, 401 insertions(+), 188 deletions(-) diff --git a/kernel/multikernel/core.c b/kernel/multikernel/core.c index 6653e212589ec5..eff8380ec835c0 100644 --- a/kernel/multikernel/core.c +++ b/kernel/multikernel/core.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -13,7 +14,9 @@ #include #include #include +#include #include +#include #include "internal.h" /* Lock order: transaction -> route write -> ownership -> resources. */ @@ -1538,10 +1541,12 @@ void mk_kimage_free(struct kimage *image, void *virt_addr, size_t size) struct mk_shutdown_work { struct work_struct work; - u32 flags; int sender_instance_id; }; +static struct mk_shutdown_work mk_shutdown_work; +static atomic_t mk_shutdown_work_pending = ATOMIC_INIT(0); + /* * Notify @target_id that this kernel is going down, while messaging @@ -1591,10 +1596,10 @@ void __noreturn mk_halt_to_pool(void) static void mk_shutdown_work_fn(struct work_struct *work) { - struct mk_shutdown_work *sw = container_of(work, struct mk_shutdown_work, work); + struct mk_shutdown_work *sw = + container_of(work, struct mk_shutdown_work, work); int sender_instance_id = sw->sender_instance_id; - kfree(sw); mk_notify_down_and_park(sender_instance_id, MK_SYS_SHUTDOWN_ACK); } @@ -1628,9 +1633,49 @@ static int mk_instance_settle_halted(struct mk_instance *instance) struct mk_halted_work { struct work_struct work; + unsigned int pool_slot; int instance_id; }; +static struct mk_halted_work *mk_halted_work_pool; +static unsigned long *mk_halted_work_pool_busy; +static unsigned int mk_halted_work_pool_size; +static atomic_t mk_halted_work_pool_cursor = ATOMIC_INIT(0); +static struct workqueue_struct *mk_system_workqueue; +static atomic64_t mk_system_work_dropped = ATOMIC64_INIT(0); +static DEFINE_RATELIMIT_STATE(mk_system_work_rs, + DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); + +static void mk_system_work_drop(void) +{ + atomic64_inc(&mk_system_work_dropped); + if (__ratelimit(&mk_system_work_rs)) + printk_deferred("WARNING: Multikernel system work request dropped\n"); +} + +static struct mk_halted_work *mk_halted_work_get(void) +{ + unsigned int start; + unsigned int slot; + unsigned int i; + + start = (unsigned int)atomic_inc_return(&mk_halted_work_pool_cursor); + for (i = 0; i < mk_halted_work_pool_size; i++) { + slot = (start + i) % mk_halted_work_pool_size; + if (!test_and_set_bit_lock(slot, mk_halted_work_pool_busy)) + return &mk_halted_work_pool[slot]; + } + + mk_system_work_drop(); + return NULL; +} + +static void mk_halted_work_put(struct mk_halted_work *aw) +{ + clear_bit_unlock(aw->pool_slot, mk_halted_work_pool_busy); +} + static void mk_halted_work_fn(struct work_struct *work) { struct mk_halted_work *aw = @@ -1648,7 +1693,7 @@ static void mk_halted_work_fn(struct work_struct *work) aw->instance_id); } - kfree(aw); + mk_halted_work_put(aw); } static void mk_system_msg_handler(u32 msg_type, u32 subtype, @@ -1660,22 +1705,19 @@ static void mk_system_msg_handler(u32 msg_type, u32 subtype, switch (subtype) { case MK_SYS_SHUTDOWN: { - struct mk_shutdown_payload *req = payload; - struct mk_shutdown_work *sw; - - if (payload_len < sizeof(*req)) + if (payload_len < sizeof(struct mk_shutdown_payload)) return; pr_info("Shutdown requested by instance %d\n", sender_instance_id); - sw = kmalloc(sizeof(*sw), GFP_ATOMIC); - if (!sw) + if (atomic_cmpxchg(&mk_shutdown_work_pending, 0, 1)) { + mk_system_work_drop(); return; + } - INIT_WORK(&sw->work, mk_shutdown_work_fn); - sw->flags = req->flags; - sw->sender_instance_id = sender_instance_id; - schedule_work(&sw->work); + mk_shutdown_work.sender_instance_id = sender_instance_id; + if (!queue_work(mk_system_workqueue, &mk_shutdown_work.work)) + mk_system_work_drop(); break; } case MK_SYS_SHUTDOWN_ACK: { @@ -1710,13 +1752,15 @@ static void mk_system_msg_handler(u32 msg_type, u32 subtype, * need time to reach the park loop, while this runs in IPI * context. */ - aw = kmalloc(sizeof(*aw), GFP_ATOMIC); + aw = mk_halted_work_get(); if (!aw) break; - INIT_WORK(&aw->work, mk_halted_work_fn); - aw->instance_id = ack->resource_id; - schedule_work(&aw->work); + aw->instance_id = sender_instance_id; + if (!queue_work(mk_system_workqueue, &aw->work)) { + mk_halted_work_put(aw); + mk_system_work_drop(); + } break; } default: @@ -2034,6 +2078,65 @@ int multikernel_force_halt_by_id(int mk_id) return ret; } +static int __init mk_system_work_init(void) +{ + unsigned int pool_size; + unsigned int i; + + INIT_WORK(&mk_shutdown_work.work, mk_shutdown_work_fn); + atomic_set(&mk_shutdown_work_pending, 0); + + /* Allow every possible CPU to have all reply slots in flight. */ + pool_size = max_t(unsigned int, num_possible_cpus(), 1) * + MK_REPLY_SLOTS; + mk_halted_work_pool = kcalloc(pool_size, sizeof(*mk_halted_work_pool), + GFP_KERNEL); + if (!mk_halted_work_pool) + return -ENOMEM; + + mk_halted_work_pool_busy = bitmap_zalloc(pool_size, GFP_KERNEL); + if (!mk_halted_work_pool_busy) + goto free_pool; + mk_halted_work_pool_size = pool_size; + atomic_set(&mk_halted_work_pool_cursor, 0); + for (i = 0; i < pool_size; i++) { + INIT_WORK(&mk_halted_work_pool[i].work, mk_halted_work_fn); + mk_halted_work_pool[i].pool_slot = i; + } + + mk_system_workqueue = + alloc_workqueue("mk-system", WQ_UNBOUND | WQ_MEM_RECLAIM, 0); + if (!mk_system_workqueue) + goto free_pool_bitmap; + + return 0; + +free_pool_bitmap: + bitmap_free(mk_halted_work_pool_busy); + mk_halted_work_pool_busy = NULL; + mk_halted_work_pool_size = 0; +free_pool: + kfree(mk_halted_work_pool); + mk_halted_work_pool = NULL; + return -ENOMEM; +} + +static void mk_system_work_cleanup(void) +{ + if (mk_system_workqueue) { + destroy_workqueue(mk_system_workqueue); + mk_system_workqueue = NULL; + } + WARN_ON_ONCE(mk_halted_work_pool_busy && + !bitmap_empty(mk_halted_work_pool_busy, + mk_halted_work_pool_size)); + bitmap_free(mk_halted_work_pool_busy); + mk_halted_work_pool_busy = NULL; + mk_halted_work_pool_size = 0; + kfree(mk_halted_work_pool); + mk_halted_work_pool = NULL; +} + static int __init multikernel_init(void) { int ret; @@ -2049,9 +2152,18 @@ static int __init multikernel_init(void) return ret; } + ret = mk_system_work_init(); + if (ret < 0) { + pr_err("Failed to initialize multikernel system work: %d\n", ret); + mk_messaging_cleanup(); + mk_pci_lease_system_cleanup(); + return ret; + } + ret = mk_register_msg_handler(MK_MSG_SYSTEM, mk_system_msg_handler, NULL); if (ret < 0) { pr_err("Failed to register system message handler: %d\n", ret); + mk_system_work_cleanup(); mk_messaging_cleanup(); mk_pci_lease_system_cleanup(); return ret; @@ -2061,6 +2173,7 @@ static int __init multikernel_init(void) if (ret < 0) { pr_err("Failed to initialize multikernel hotplug: %d\n", ret); mk_unregister_msg_handler(MK_MSG_SYSTEM, mk_system_msg_handler); + mk_system_work_cleanup(); mk_messaging_cleanup(); mk_pci_lease_system_cleanup(); return ret; @@ -2071,6 +2184,7 @@ static int __init multikernel_init(void) pr_err("Failed to initialize multikernel sysfs interface: %d\n", ret); mk_hotplug_cleanup(); mk_unregister_msg_handler(MK_MSG_SYSTEM, mk_system_msg_handler); + mk_system_work_cleanup(); mk_messaging_cleanup(); mk_pci_lease_system_cleanup(); return ret; diff --git a/kernel/multikernel/hotplug.c b/kernel/multikernel/hotplug.c index d9f63759fc0ca7..19e400b195e5d9 100644 --- a/kernel/multikernel/hotplug.c +++ b/kernel/multikernel/hotplug.c @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -23,6 +24,8 @@ #include #include #include +#include +#include #include "internal.h" static const char mk_mem_resource_name[] = "System RAM (multikernel)"; @@ -65,17 +68,97 @@ struct mk_hotplug_op { static DEFINE_MUTEX(mk_hotplug_mutex); static LIST_HEAD(mk_hotplug_ops); +struct mk_hotplug_work { + struct work_struct work; + work_func_t dispatch; + unsigned int pool_slot; + int sender_instance_id; + u32 operation; + union { + struct { + mk_phys_cpu_t cpu_id; + u32 numa_node; + u32 flags; + } cpu; + struct { + u64 start_pfn; + u64 nr_pages; + u32 numa_node; + u32 mem_type; + } mem; + struct { + u16 domain; + u8 bus; + u8 devfn; + u32 flags; + char driver_override[64]; + } device; + }; +}; + +static struct mk_hotplug_work *mk_hotplug_work_pool; +static unsigned long *mk_hotplug_work_pool_busy; +static unsigned int mk_hotplug_work_pool_size; +static atomic_t mk_hotplug_work_pool_cursor = ATOMIC_INIT(0); +static struct workqueue_struct *mk_hotplug_workqueue; +static atomic64_t mk_hotplug_work_dropped = ATOMIC64_INIT(0); +static DEFINE_RATELIMIT_STATE(mk_hotplug_work_pool_rs, + DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); + +static void mk_hotplug_work_drop(void) +{ + atomic64_inc(&mk_hotplug_work_dropped); + if (__ratelimit(&mk_hotplug_work_pool_rs)) + printk_deferred("WARNING: Multikernel hotplug work request dropped\n"); +} + +static struct mk_hotplug_work *mk_hotplug_work_get(void) +{ + unsigned int start; + unsigned int slot; + unsigned int i; + + start = (unsigned int)atomic_inc_return(&mk_hotplug_work_pool_cursor); + for (i = 0; i < mk_hotplug_work_pool_size; i++) { + slot = (start + i) % mk_hotplug_work_pool_size; + if (!test_and_set_bit_lock(slot, mk_hotplug_work_pool_busy)) + return &mk_hotplug_work_pool[slot]; + } + + mk_hotplug_work_drop(); + return NULL; +} + +static void mk_hotplug_work_put(struct mk_hotplug_work *hp_work) +{ + clear_bit_unlock(hp_work->pool_slot, mk_hotplug_work_pool_busy); +} + +static void mk_hotplug_work_fn(struct work_struct *work) +{ + struct mk_hotplug_work *hp_work = + container_of(work, struct mk_hotplug_work, work); + + hp_work->dispatch(work); + mk_hotplug_work_put(hp_work); +} + +static int mk_hotplug_work_queue(struct mk_hotplug_work *hp_work, + work_func_t dispatch) +{ + hp_work->dispatch = dispatch; + if (queue_work(mk_hotplug_workqueue, &hp_work->work)) + return 0; + + mk_hotplug_work_put(hp_work); + mk_hotplug_work_drop(); + return -EBUSY; +} + /* * CPU Hotplug Operations */ -struct mk_cpu_hotplug_work { - struct work_struct work; - mk_phys_cpu_t cpu_id; - u32 numa_node; - u32 flags; - int sender_instance_id; /* For sending ACK back */ - u32 operation; /* MK_RES_CPU_ADD or MK_RES_CPU_REMOVE */ -}; /** * Search present CPUs (not possible CPUs) to find the logical CPU with matching @@ -285,122 +368,113 @@ static int mk_do_cpu_remove(mk_phys_cpu_t cpu_id) static void mk_cpu_add_work_fn(struct work_struct *work) { - struct mk_cpu_hotplug_work *hp_work = container_of(work, struct mk_cpu_hotplug_work, work); + struct mk_hotplug_work *hp_work = + container_of(work, struct mk_hotplug_work, work); struct mk_resource_ack ack; int ret, ack_ret; - ret = mk_do_cpu_add(hp_work->cpu_id, hp_work->numa_node, hp_work->flags); + ret = mk_do_cpu_add(hp_work->cpu.cpu_id, hp_work->cpu.numa_node, + hp_work->cpu.flags); /* Send ACK back to sender */ ack.operation = hp_work->operation; ack.result = ret; - ack.resource_id = hp_work->cpu_id; + ack.resource_id = hp_work->cpu.cpu_id; ack_ret = mk_send_message(hp_work->sender_instance_id, MK_MSG_RESOURCE, MK_RES_ACK, &ack, sizeof(ack)); if (ack_ret < 0) { pr_warn("Multikernel hotplug: Failed to send ACK for CPU %llu: %d\n", - hp_work->cpu_id, ack_ret); + hp_work->cpu.cpu_id, ack_ret); } - - kfree(hp_work); } static void mk_cpu_remove_work_fn(struct work_struct *work) { - struct mk_cpu_hotplug_work *hp_work = container_of(work, struct mk_cpu_hotplug_work, work); + struct mk_hotplug_work *hp_work = + container_of(work, struct mk_hotplug_work, work); struct mk_resource_ack ack; int ret, ack_ret; - ret = mk_do_cpu_remove(hp_work->cpu_id); + ret = mk_do_cpu_remove(hp_work->cpu.cpu_id); /* Send ACK back to sender */ ack.operation = hp_work->operation; ack.result = ret; - ack.resource_id = hp_work->cpu_id; + ack.resource_id = hp_work->cpu.cpu_id; ack_ret = mk_send_message(hp_work->sender_instance_id, MK_MSG_RESOURCE, MK_RES_ACK, &ack, sizeof(ack)); if (ack_ret < 0) { pr_warn("Multikernel hotplug: Failed to send ACK for CPU %llu: %d\n", - hp_work->cpu_id, ack_ret); + hp_work->cpu.cpu_id, ack_ret); } - - kfree(hp_work); } /** * mk_handle_cpu_add - Handle CPU addition request * @payload: CPU resource payload * @payload_len: Payload length + * @sender_instance_id: Authenticated sending instance * * Brings a CPU online in the receiving kernel. This is called on the spawn * kernel side when the host kernel transfers a CPU to it. * * Returns 0 on success, negative error code on failure */ -static int mk_handle_cpu_add(struct mk_cpu_resource_payload *payload, u32 payload_len) +static int mk_handle_cpu_add(struct mk_cpu_resource_payload *payload, + u32 payload_len, s32 sender_instance_id) { - struct mk_cpu_hotplug_work *hp_work; + struct mk_hotplug_work *hp_work; if (payload_len < sizeof(*payload)) { pr_err("Multikernel hotplug: Invalid CPU add payload size: %u\n", payload_len); return -EINVAL; } - /* - * We're in IRQ context (IPI handler), so we can't call add_cpu() directly. - */ - hp_work = kmalloc(sizeof(*hp_work), GFP_ATOMIC); + hp_work = mk_hotplug_work_get(); if (!hp_work) return -ENOMEM; - INIT_WORK(&hp_work->work, mk_cpu_add_work_fn); - hp_work->cpu_id = payload->cpu_id; - hp_work->numa_node = payload->numa_node; - hp_work->flags = payload->flags; - hp_work->sender_instance_id = payload->sender_instance_id; + hp_work->cpu.cpu_id = payload->cpu_id; + hp_work->cpu.numa_node = payload->numa_node; + hp_work->cpu.flags = payload->flags; + hp_work->sender_instance_id = sender_instance_id; hp_work->operation = MK_RES_CPU_ADD; - schedule_work(&hp_work->work); - - return 0; + return mk_hotplug_work_queue(hp_work, mk_cpu_add_work_fn); } /** * mk_handle_cpu_remove - Handle CPU removal request * @payload: CPU resource payload * @payload_len: Payload length + * @sender_instance_id: Authenticated sending instance * * Takes a CPU offline in the current kernel. This is called on the host * kernel side before transferring a CPU to a spawn kernel. * * Returns 0 on success, negative error code on failure */ -int mk_handle_cpu_remove(struct mk_cpu_resource_payload *payload, u32 payload_len) +int mk_handle_cpu_remove(struct mk_cpu_resource_payload *payload, + u32 payload_len, s32 sender_instance_id) { - struct mk_cpu_hotplug_work *hp_work; + struct mk_hotplug_work *hp_work; if (payload_len < sizeof(*payload)) { pr_err("Multikernel hotplug: Invalid CPU remove payload size: %u\n", payload_len); return -EINVAL; } - /* - * We're in IRQ context (IPI handler), so we can't call remove_cpu() directly. - */ - hp_work = kmalloc(sizeof(*hp_work), GFP_ATOMIC); + hp_work = mk_hotplug_work_get(); if (!hp_work) return -ENOMEM; - INIT_WORK(&hp_work->work, mk_cpu_remove_work_fn); - hp_work->cpu_id = payload->cpu_id; - hp_work->numa_node = payload->numa_node; - hp_work->flags = payload->flags; - hp_work->sender_instance_id = payload->sender_instance_id; + hp_work->cpu.cpu_id = payload->cpu_id; + hp_work->cpu.numa_node = payload->numa_node; + hp_work->cpu.flags = payload->flags; + hp_work->sender_instance_id = sender_instance_id; hp_work->operation = MK_RES_CPU_REMOVE; - schedule_work(&hp_work->work); - - return 0; + return mk_hotplug_work_queue(hp_work, mk_cpu_remove_work_fn); } /* @@ -559,113 +633,91 @@ static int mk_do_mem_remove(u64 start_pfn, u64 nr_pages) return 0; } -struct mk_mem_hotplug_work { - struct work_struct work; - u64 start_pfn; - u64 nr_pages; - u32 numa_node; - u32 mem_type; - int sender_instance_id; - u32 operation; -}; - static void mk_mem_add_work_fn(struct work_struct *work) { - struct mk_mem_hotplug_work *hp_work = container_of(work, struct mk_mem_hotplug_work, work); + struct mk_hotplug_work *hp_work = + container_of(work, struct mk_hotplug_work, work); struct mk_resource_ack ack; int ret, ack_ret; - ret = mk_do_mem_add(hp_work->start_pfn, hp_work->nr_pages, - hp_work->numa_node, hp_work->mem_type); + ret = mk_do_mem_add(hp_work->mem.start_pfn, hp_work->mem.nr_pages, + hp_work->mem.numa_node, hp_work->mem.mem_type); ack.operation = hp_work->operation; ack.result = ret; - ack.resource_id = hp_work->start_pfn; + ack.resource_id = hp_work->mem.start_pfn; ack_ret = mk_send_message(hp_work->sender_instance_id, MK_MSG_RESOURCE, MK_RES_ACK, &ack, sizeof(ack)); if (ack_ret < 0) { pr_warn("Multikernel hotplug: Failed to send ACK for mem add at 0x%llx: %d\n", - (u64)hp_work->start_pfn, ack_ret); + hp_work->mem.start_pfn, ack_ret); } - - kfree(hp_work); } static void mk_mem_remove_work_fn(struct work_struct *work) { - struct mk_mem_hotplug_work *hp_work = container_of(work, struct mk_mem_hotplug_work, work); + struct mk_hotplug_work *hp_work = + container_of(work, struct mk_hotplug_work, work); struct mk_resource_ack ack; int ret, ack_ret; - ret = mk_do_mem_remove(hp_work->start_pfn, hp_work->nr_pages); + ret = mk_do_mem_remove(hp_work->mem.start_pfn, hp_work->mem.nr_pages); ack.operation = hp_work->operation; ack.result = ret; - ack.resource_id = hp_work->start_pfn; + ack.resource_id = hp_work->mem.start_pfn; ack_ret = mk_send_message(hp_work->sender_instance_id, MK_MSG_RESOURCE, MK_RES_ACK, &ack, sizeof(ack)); if (ack_ret < 0) { pr_warn("Multikernel hotplug: Failed to send ACK for mem remove at 0x%llx: %d\n", - (u64)hp_work->start_pfn, ack_ret); + hp_work->mem.start_pfn, ack_ret); } - - kfree(hp_work); } -static int mk_handle_mem_add(struct mk_mem_resource_payload *payload, u32 payload_len) +static int mk_handle_mem_add(struct mk_mem_resource_payload *payload, + u32 payload_len, s32 sender_instance_id) { - struct mk_mem_hotplug_work *hp_work; + struct mk_hotplug_work *hp_work; if (payload_len < sizeof(*payload)) { pr_err("Multikernel hotplug: Invalid memory add payload size: %u\n", payload_len); return -EINVAL; } - hp_work = kmalloc(sizeof(*hp_work), GFP_ATOMIC); - if (!hp_work) { - pr_err("Multikernel hotplug: Failed to allocate work for mem add at 0x%llx\n", - (u64)payload->start_pfn); + hp_work = mk_hotplug_work_get(); + if (!hp_work) return -ENOMEM; - } - INIT_WORK(&hp_work->work, mk_mem_add_work_fn); - hp_work->start_pfn = payload->start_pfn; - hp_work->nr_pages = payload->nr_pages; - hp_work->numa_node = payload->numa_node; - hp_work->mem_type = payload->mem_type; - hp_work->sender_instance_id = payload->sender_instance_id; + hp_work->mem.start_pfn = payload->start_pfn; + hp_work->mem.nr_pages = payload->nr_pages; + hp_work->mem.numa_node = payload->numa_node; + hp_work->mem.mem_type = payload->mem_type; + hp_work->sender_instance_id = sender_instance_id; hp_work->operation = MK_RES_MEM_ADD; - schedule_work(&hp_work->work); - - return 0; + return mk_hotplug_work_queue(hp_work, mk_mem_add_work_fn); } -static int mk_handle_mem_remove(struct mk_mem_resource_payload *payload, u32 payload_len) +static int mk_handle_mem_remove(struct mk_mem_resource_payload *payload, + u32 payload_len, s32 sender_instance_id) { - struct mk_mem_hotplug_work *hp_work; + struct mk_hotplug_work *hp_work; if (payload_len < sizeof(*payload)) { pr_err("Multikernel hotplug: Invalid memory remove payload size: %u\n", payload_len); return -EINVAL; } - hp_work = kmalloc(sizeof(*hp_work), GFP_ATOMIC); - if (!hp_work) { - pr_err("Multikernel hotplug: Failed to allocate work for mem remove at 0x%llx\n", - (u64)payload->start_pfn); + hp_work = mk_hotplug_work_get(); + if (!hp_work) return -ENOMEM; - } - INIT_WORK(&hp_work->work, mk_mem_remove_work_fn); - hp_work->start_pfn = payload->start_pfn; - hp_work->nr_pages = payload->nr_pages; - hp_work->sender_instance_id = payload->sender_instance_id; + hp_work->mem.start_pfn = payload->start_pfn; + hp_work->mem.nr_pages = payload->nr_pages; + hp_work->sender_instance_id = sender_instance_id; hp_work->operation = MK_RES_MEM_REMOVE; - schedule_work(&hp_work->work); - - return 0; + return mk_hotplug_work_queue(hp_work, mk_mem_remove_work_fn); } /* @@ -855,121 +907,104 @@ static int mk_do_device_remove(u16 domain, u8 bus, u8 devfn) } #endif /* CONFIG_PCI */ -struct mk_device_hotplug_work { - struct work_struct work; - u16 domain; - u8 bus; - u8 devfn; - u32 flags; - char driver_override[64]; - int sender_instance_id; - u32 operation; -}; - static void mk_device_add_work_fn(struct work_struct *work) { - struct mk_device_hotplug_work *hp_work = container_of(work, struct mk_device_hotplug_work, work); + struct mk_hotplug_work *hp_work = + container_of(work, struct mk_hotplug_work, work); struct mk_resource_ack ack; int ret, ack_ret; - ret = mk_do_device_add(hp_work->domain, hp_work->bus, hp_work->devfn, - hp_work->driver_override, hp_work->flags); + ret = mk_do_device_add(hp_work->device.domain, hp_work->device.bus, + hp_work->device.devfn, + hp_work->device.driver_override, + hp_work->device.flags); ack.operation = hp_work->operation; ack.result = ret; - ack.resource_id = (hp_work->domain << 16) | (hp_work->bus << 8) | hp_work->devfn; + ack.resource_id = (hp_work->device.domain << 16) | + (hp_work->device.bus << 8) | hp_work->device.devfn; ack_ret = mk_send_message(hp_work->sender_instance_id, MK_MSG_RESOURCE, MK_RES_ACK, &ack, sizeof(ack)); if (ack_ret < 0) { pr_warn("Multikernel hotplug: Failed to send ACK for device %04x:%02x:%02x.%x: %d\n", - hp_work->domain, hp_work->bus, - PCI_SLOT(hp_work->devfn), PCI_FUNC(hp_work->devfn), ack_ret); + hp_work->device.domain, hp_work->device.bus, + PCI_SLOT(hp_work->device.devfn), + PCI_FUNC(hp_work->device.devfn), ack_ret); } - - kfree(hp_work); } static void mk_device_remove_work_fn(struct work_struct *work) { - struct mk_device_hotplug_work *hp_work = container_of(work, struct mk_device_hotplug_work, work); + struct mk_hotplug_work *hp_work = + container_of(work, struct mk_hotplug_work, work); struct mk_resource_ack ack; int ret, ack_ret; - ret = mk_do_device_remove(hp_work->domain, hp_work->bus, hp_work->devfn); + ret = mk_do_device_remove(hp_work->device.domain, hp_work->device.bus, + hp_work->device.devfn); ack.operation = hp_work->operation; ack.result = ret; - ack.resource_id = (hp_work->domain << 16) | (hp_work->bus << 8) | hp_work->devfn; + ack.resource_id = (hp_work->device.domain << 16) | + (hp_work->device.bus << 8) | hp_work->device.devfn; ack_ret = mk_send_message(hp_work->sender_instance_id, MK_MSG_RESOURCE, MK_RES_ACK, &ack, sizeof(ack)); if (ack_ret < 0) { pr_warn("Multikernel hotplug: Failed to send ACK for device %04x:%02x:%02x.%x: %d\n", - hp_work->domain, hp_work->bus, - PCI_SLOT(hp_work->devfn), PCI_FUNC(hp_work->devfn), ack_ret); + hp_work->device.domain, hp_work->device.bus, + PCI_SLOT(hp_work->device.devfn), + PCI_FUNC(hp_work->device.devfn), ack_ret); } - - kfree(hp_work); } -static int mk_handle_device_add(struct mk_device_resource_payload *payload, u32 payload_len) +static int mk_handle_device_add(struct mk_device_resource_payload *payload, + u32 payload_len, s32 sender_instance_id) { - struct mk_device_hotplug_work *hp_work; + struct mk_hotplug_work *hp_work; if (payload_len < sizeof(*payload)) { pr_err("Multikernel hotplug: Invalid device add payload size: %u\n", payload_len); return -EINVAL; } - hp_work = kmalloc(sizeof(*hp_work), GFP_ATOMIC); - if (!hp_work) { - pr_err("Multikernel hotplug: Failed to allocate work structure for device %04x:%02x:%02x.%x\n", - payload->domain, payload->bus, - PCI_SLOT(payload->devfn), PCI_FUNC(payload->devfn)); + hp_work = mk_hotplug_work_get(); + if (!hp_work) return -ENOMEM; - } - INIT_WORK(&hp_work->work, mk_device_add_work_fn); - hp_work->domain = payload->domain; - hp_work->bus = payload->bus; - hp_work->devfn = payload->devfn; - hp_work->flags = payload->flags; - strscpy(hp_work->driver_override, payload->driver_override, sizeof(hp_work->driver_override)); - hp_work->sender_instance_id = payload->sender_instance_id; + hp_work->device.domain = payload->domain; + hp_work->device.bus = payload->bus; + hp_work->device.devfn = payload->devfn; + hp_work->device.flags = payload->flags; + strscpy(hp_work->device.driver_override, payload->driver_override, + sizeof(hp_work->device.driver_override)); + hp_work->sender_instance_id = sender_instance_id; hp_work->operation = MK_RES_DEVICE_ADD; - schedule_work(&hp_work->work); - - return 0; + return mk_hotplug_work_queue(hp_work, mk_device_add_work_fn); } -static int mk_handle_device_remove(struct mk_device_resource_payload *payload, u32 payload_len) +static int mk_handle_device_remove(struct mk_device_resource_payload *payload, + u32 payload_len, s32 sender_instance_id) { - struct mk_device_hotplug_work *hp_work; + struct mk_hotplug_work *hp_work; if (payload_len < sizeof(*payload)) { pr_err("Multikernel hotplug: Invalid device remove payload size: %u\n", payload_len); return -EINVAL; } - hp_work = kmalloc(sizeof(*hp_work), GFP_ATOMIC); - if (!hp_work) { - pr_err("Multikernel hotplug: Failed to allocate work structure for device %04x:%02x:%02x.%x\n", - payload->domain, payload->bus, - PCI_SLOT(payload->devfn), PCI_FUNC(payload->devfn)); + hp_work = mk_hotplug_work_get(); + if (!hp_work) return -ENOMEM; - } - INIT_WORK(&hp_work->work, mk_device_remove_work_fn); - hp_work->domain = payload->domain; - hp_work->bus = payload->bus; - hp_work->devfn = payload->devfn; - hp_work->flags = payload->flags; - hp_work->sender_instance_id = payload->sender_instance_id; + hp_work->device.domain = payload->domain; + hp_work->device.bus = payload->bus; + hp_work->device.devfn = payload->devfn; + hp_work->device.flags = payload->flags; + hp_work->sender_instance_id = sender_instance_id; hp_work->operation = MK_RES_DEVICE_REMOVE; - schedule_work(&hp_work->work); - - return 0; + return mk_hotplug_work_queue(hp_work, mk_device_remove_work_fn); } /* @@ -982,6 +1017,7 @@ static int mk_handle_device_remove(struct mk_device_resource_payload *payload, u * @subtype: Message subtype (CPU_ADD, CPU_REMOVE, MEM_ADD, MEM_REMOVE, DEVICE_ADD, DEVICE_REMOVE) * @payload: Payload data * @payload_len: Payload length + * @sender_instance_id: Authenticated sending instance * @ctx: Context (unused) * * This is the main message handler registered with the multikernel @@ -1001,27 +1037,31 @@ static void mk_resource_msg_handler(u32 msg_type, u32 subtype, switch (subtype) { case MK_RES_CPU_ADD: - ret = mk_handle_cpu_add((struct mk_cpu_resource_payload *)payload, payload_len); + ret = mk_handle_cpu_add(payload, payload_len, sender_instance_id); break; case MK_RES_CPU_REMOVE: - ret = mk_handle_cpu_remove((struct mk_cpu_resource_payload *)payload, payload_len); + ret = mk_handle_cpu_remove(payload, payload_len, + sender_instance_id); break; case MK_RES_MEM_ADD: - ret = mk_handle_mem_add((struct mk_mem_resource_payload *)payload, payload_len); + ret = mk_handle_mem_add(payload, payload_len, sender_instance_id); break; case MK_RES_MEM_REMOVE: - ret = mk_handle_mem_remove((struct mk_mem_resource_payload *)payload, payload_len); + ret = mk_handle_mem_remove(payload, payload_len, + sender_instance_id); break; case MK_RES_DEVICE_ADD: - ret = mk_handle_device_add((struct mk_device_resource_payload *)payload, payload_len); + ret = mk_handle_device_add(payload, payload_len, + sender_instance_id); break; case MK_RES_DEVICE_REMOVE: - ret = mk_handle_device_remove((struct mk_device_resource_payload *)payload, payload_len); + ret = mk_handle_device_remove(payload, payload_len, + sender_instance_id); break; case MK_RES_ACK: @@ -1063,19 +1103,65 @@ static void mk_resource_msg_handler(u32 msg_type, u32 subtype, */ int __init mk_hotplug_init(void) { + unsigned int pool_size; + unsigned int i; int ret; pr_info("Initializing multikernel hotplug subsystem\n"); + /* + * Match the PCI control-plane reserve. Exhaustion is still handled + * explicitly because resource requests do not consume reply-slot credits. + */ + pool_size = max_t(unsigned int, num_possible_cpus(), 1) * + MK_REPLY_SLOTS; + mk_hotplug_work_pool = kcalloc(pool_size, + sizeof(*mk_hotplug_work_pool), + GFP_KERNEL); + if (!mk_hotplug_work_pool) + return -ENOMEM; + + mk_hotplug_work_pool_busy = bitmap_zalloc(pool_size, GFP_KERNEL); + if (!mk_hotplug_work_pool_busy) { + ret = -ENOMEM; + goto free_pool; + } + mk_hotplug_work_pool_size = pool_size; + atomic_set(&mk_hotplug_work_pool_cursor, 0); + for (i = 0; i < pool_size; i++) { + INIT_WORK(&mk_hotplug_work_pool[i].work, + mk_hotplug_work_fn); + mk_hotplug_work_pool[i].pool_slot = i; + } + + mk_hotplug_workqueue = + alloc_workqueue("mk-hotplug", WQ_UNBOUND | WQ_MEM_RECLAIM, 0); + if (!mk_hotplug_workqueue) { + ret = -ENOMEM; + goto free_pool_bitmap; + } + /* Register handler for resource management messages */ ret = mk_register_msg_handler(MK_MSG_RESOURCE, mk_resource_msg_handler, NULL); if (ret < 0) { pr_err("Failed to register multikernel hotplug message handler: %d\n", ret); - return ret; + goto destroy_workqueue; } pr_info("Multikernel hotplug subsystem initialized\n"); return 0; + +destroy_workqueue: + destroy_workqueue(mk_hotplug_workqueue); + mk_hotplug_workqueue = NULL; +free_pool_bitmap: + bitmap_free(mk_hotplug_work_pool_busy); + mk_hotplug_work_pool_busy = NULL; + mk_hotplug_work_pool_size = 0; +free_pool: + kfree(mk_hotplug_work_pool); + mk_hotplug_work_pool = NULL; + return ret; } /** @@ -1091,6 +1177,18 @@ void mk_hotplug_cleanup(void) /* Unregister message handler */ mk_unregister_msg_handler(MK_MSG_RESOURCE, mk_resource_msg_handler); + if (mk_hotplug_workqueue) { + destroy_workqueue(mk_hotplug_workqueue); + mk_hotplug_workqueue = NULL; + } + WARN_ON_ONCE(mk_hotplug_work_pool_busy && + !bitmap_empty(mk_hotplug_work_pool_busy, + mk_hotplug_work_pool_size)); + bitmap_free(mk_hotplug_work_pool_busy); + mk_hotplug_work_pool_busy = NULL; + mk_hotplug_work_pool_size = 0; + kfree(mk_hotplug_work_pool); + mk_hotplug_work_pool = NULL; /* Free operation tracking list */ mutex_lock(&mk_hotplug_mutex); diff --git a/kernel/multikernel/internal.h b/kernel/multikernel/internal.h index d770ef65650814..8926eb181bdb9f 100644 --- a/kernel/multikernel/internal.h +++ b/kernel/multikernel/internal.h @@ -145,7 +145,8 @@ int mk_arm_force_halt(struct mk_instance *instance); /* hotplug.c */ int mk_hotplug_init(void); void mk_hotplug_cleanup(void); -int mk_handle_cpu_remove(struct mk_cpu_resource_payload *payload, u32 payload_len); +int mk_handle_cpu_remove(struct mk_cpu_resource_payload *payload, + u32 payload_len, s32 sender_instance_id); /* * Move primitives between this kernel and the pool it manages. Valid