From 7ead5a3ca9860356e88675e3ccfb2d084ae3a26b Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Fri, 2 Oct 2026 18:23:56 -0400 Subject: [PATCH 01/50] feat(actor): preempt CPU-bound Lua actors with a tick budget A Lua actor in a long loop held its scheduler worker until it blocked or finished, starving every other actor on that worker. Lua actors now run each step under a VM tick budget (go-lua SetTickBudget) and are suspended at a safepoint when it is used up. - process.Preemptible (EnablePreemption) and StepPreempted. The actor scheduler enables preemption at admission and requeues a preempted process through the global FIFO behind other ready work, also after dispatching the same step's yields. Pool, Temporal and eval drivers do not enable it and are unchanged. - The Lua process spends one budget per step across all its coroutines; tasks that did not finish run in the next step. - Options match v2: tick_budget (0 = default 512, negative = unlimited) and max_steps (0 = unlimited, exceeding fails with process.ErrStepLimitExceeded), read from entry meta.options and overridden by process.with_options. Keys directly under meta are rejected; with_options validates values at the call site. - go-lua bumped to the preemption commit (wippyai/go-lua#60). --- api/process/errors.go | 3 + api/process/preempt.go | 12 + api/process/step.go | 10 + api/registry/options.go | 17 + api/runtime/lua/options.go | 129 +++++++ api/runtime/lua/options_test.go | 72 ++++ go.mod | 2 +- go.sum | 2 + runtime/lua/component/process/manager.go | 48 ++- runtime/lua/component/process/manager_test.go | 2 +- runtime/lua/engine/preempt_test.go | 347 ++++++++++++++++++ runtime/lua/engine/process.go | 117 +++++- runtime/lua/engine/process_factory.go | 24 +- runtime/lua/modules/process/context.go | 6 + .../modules/process/spawner_security_test.go | 12 + system/scheduler/actor/preempt_test.go | 198 ++++++++++ system/scheduler/actor/scheduler.go | 20 + system/scheduler/actor/worker.go | 19 +- 18 files changed, 1017 insertions(+), 23 deletions(-) create mode 100644 api/process/preempt.go create mode 100644 api/registry/options.go create mode 100644 api/runtime/lua/options.go create mode 100644 api/runtime/lua/options_test.go create mode 100644 runtime/lua/engine/preempt_test.go create mode 100644 system/scheduler/actor/preempt_test.go diff --git a/api/process/errors.go b/api/process/errors.go index 2658d08be..9de9b6999 100644 --- a/api/process/errors.go +++ b/api/process/errors.go @@ -20,6 +20,9 @@ const ( var ( ErrMaxProcessesExceeded = apierror.New(LimitExceeded, "max processes limit exceeded").WithRetryable(apierror.False) + // ErrStepLimitExceeded fails a process that exceeds its max_steps option. + ErrStepLimitExceeded = apierror.New(apierror.RateLimited, "process step limit exceeded").WithRetryable(apierror.False) + ErrProcessClosed = apierror.New(InvalidState, "process closed").WithRetryable(apierror.False) ErrProcessNotFound = apierror.New(NotFound, "process not found").WithRetryable(apierror.False) diff --git a/api/process/preempt.go b/api/process/preempt.go new file mode 100644 index 000000000..026471618 --- /dev/null +++ b/api/process/preempt.go @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: MPL-2.0 + +package process + +// Preemptible is an optional interface for processes that can suspend a +// running step at a safepoint once they have used up their execution slice. +// A scheduler calls EnablePreemption when it runs the process again after +// Step reports StepPreempted; a process that was never enabled runs each step +// to a blocking point or completion. +type Preemptible interface { + EnablePreemption() +} diff --git a/api/process/step.go b/api/process/step.go index 44e9d3c0c..093f996e1 100644 --- a/api/process/step.go +++ b/api/process/step.go @@ -15,6 +15,10 @@ const ( StepDone StepYield // has yields to dispatch, wait for completions StepUpgrade // process requested upgrade, worker handles swap + // StepPreempted reports that the process used up its preemption budget + // and is ready to continue. The scheduler dispatches the step's yields + // and runs the process again after other ready work. + StepPreempted ) // MaxYields is the maximum yields per step that fit in the fixed buffer. @@ -97,6 +101,12 @@ func (o *StepOutput) Continue() { o.status = StepContinue } +// Preempt marks process as preempted: ready to continue once other ready +// work has had a turn. +func (o *StepOutput) Preempt() { + o.status = StepPreempted +} + // Reset clears output for reuse. func (o *StepOutput) Reset() { o.buf[0] = Yield{} diff --git a/api/registry/options.go b/api/registry/options.go new file mode 100644 index 000000000..0e1aeb5dd --- /dev/null +++ b/api/registry/options.go @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: MPL-2.0 + +package registry + +import "github.com/wippyai/runtime/api/attrs" + +// EntryOptionsMetaKey is the registry metadata bag holding declaration-time +// runtime options. Entry data remains the executable/config payload. +const EntryOptionsMetaKey = "options" + +// EntryOptionsFromMeta returns the nested meta.options bag. +func EntryOptionsFromMeta(meta attrs.Bag) (attrs.Bag, bool) { + if meta == nil { + return nil, false + } + return meta.GetBag(EntryOptionsMetaKey) +} diff --git a/api/runtime/lua/options.go b/api/runtime/lua/options.go new file mode 100644 index 000000000..437b78125 --- /dev/null +++ b/api/runtime/lua/options.go @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: MPL-2.0 + +package lua + +import ( + "encoding/json" + "fmt" + "math" + + "github.com/wippyai/runtime/api/attrs" + "github.com/wippyai/runtime/api/registry" +) + +// Actor execution option keys, shared by process entry meta.options and +// process spawn options. +const ( + ProcessOptionTickBudget = "tick_budget" + ProcessOptionMaxSteps = "max_steps" +) + +// ExecutionBudgets holds the execution limits of a Lua actor. +// +// TickBudget is the number of VM ticks (backward jumps, loop iterations and +// calls) an actor may run before it is preempted so other actors can run: +// zero selects the runtime default and a negative value disables preemption. +// MaxSteps bounds the number of scheduler steps the actor may take; zero is +// unlimited and exceeding it fails the actor with process.ErrStepLimitExceeded. +type ExecutionBudgets struct { + TickBudget int64 + MaxSteps uint64 + TickBudgetSet bool + MaxStepsSet bool +} + +// Override returns b with the options set in o taking precedence. +func (b ExecutionBudgets) Override(o ExecutionBudgets) ExecutionBudgets { + if o.TickBudgetSet { + b.TickBudget, b.TickBudgetSet = o.TickBudget, true + } + if o.MaxStepsSet { + b.MaxSteps, b.MaxStepsSet = o.MaxSteps, true + } + return b +} + +// ExecutionBudgetsFromOptions reads the execution options from an options +// bag. subject names the source in error messages. +func ExecutionBudgetsFromOptions(options attrs.Attributes, subject string) (ExecutionBudgets, error) { + var b ExecutionBudgets + if raw, ok := options.Get(ProcessOptionTickBudget); ok && raw != nil { + v, err := optionInt64(raw) + if err != nil { + return b, fmt.Errorf("%s %q must be an integer", subject, ProcessOptionTickBudget) + } + b.TickBudget, b.TickBudgetSet = v, true + } + if raw, ok := options.Get(ProcessOptionMaxSteps); ok && raw != nil { + v, err := optionInt64(raw) + if err != nil || v < 0 { + return b, fmt.Errorf("%s %q must be a non-negative integer", subject, ProcessOptionMaxSteps) + } + b.MaxSteps, b.MaxStepsSet = uint64(v), true + } + return b, nil +} + +// EntryExecutionBudgets reads the execution options of a process entry from +// meta.options. The options are only accepted nested under meta.options. +func EntryExecutionBudgets(meta attrs.Bag) (ExecutionBudgets, error) { + for _, key := range [...]string{ProcessOptionTickBudget, ProcessOptionMaxSteps} { + if _, ok := meta.Get(key); ok { + return ExecutionBudgets{}, fmt.Errorf("process entry meta %q must be nested under meta.%s", key, registry.EntryOptionsMetaKey) + } + } + options, ok := registry.EntryOptionsFromMeta(meta) + if !ok { + return ExecutionBudgets{}, nil + } + return ExecutionBudgetsFromOptions(options, "process entry meta.options") +} + +// maxExactFloatInteger is the largest integer a float64 holds exactly. +const maxExactFloatInteger = 1 << 53 + +func optionInt64(raw any) (int64, error) { + switch v := raw.(type) { + case int: + return int64(v), nil + case int8: + return int64(v), nil + case int16: + return int64(v), nil + case int32: + return int64(v), nil + case int64: + return v, nil + case uint: + return uintOption(uint64(v)) + case uint8: + return int64(v), nil + case uint16: + return int64(v), nil + case uint32: + return int64(v), nil + case uint64: + return uintOption(v) + case float32: + return floatOption(float64(v)) + case float64: + return floatOption(v) + case json.Number: + return v.Int64() + } + return 0, fmt.Errorf("not an integer: %T", raw) +} + +func uintOption(v uint64) (int64, error) { + if v > math.MaxInt64 { + return 0, fmt.Errorf("integer overflows int64: %d", v) + } + return int64(v), nil +} + +func floatOption(v float64) (int64, error) { + if v != math.Trunc(v) || math.Abs(v) > maxExactFloatInteger { + return 0, fmt.Errorf("not an exact integer: %v", v) + } + return int64(v), nil +} diff --git a/api/runtime/lua/options_test.go b/api/runtime/lua/options_test.go new file mode 100644 index 000000000..5238b803c --- /dev/null +++ b/api/runtime/lua/options_test.go @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: MPL-2.0 + +package lua + +import ( + "encoding/json" + "testing" + + "github.com/wippyai/runtime/api/attrs" +) + +func TestEntryExecutionBudgetsFromMetaOptions(t *testing.T) { + b, err := EntryExecutionBudgets(attrs.Bag{"options": map[string]any{"tick_budget": 2048, "max_steps": float64(10)}}) + if err != nil { + t.Fatal(err) + } + if !b.TickBudgetSet || b.TickBudget != 2048 || !b.MaxStepsSet || b.MaxSteps != 10 { + t.Fatalf("unexpected budgets %+v", b) + } + + none, err := EntryExecutionBudgets(attrs.Bag{"comment": "x"}) + if err != nil || none.TickBudgetSet || none.MaxStepsSet { + t.Fatalf("expected no budgets, got %+v %v", none, err) + } +} + +func TestEntryExecutionBudgetsRejectsDirectMetaKeys(t *testing.T) { + for _, key := range []string{ProcessOptionTickBudget, ProcessOptionMaxSteps} { + if _, err := EntryExecutionBudgets(attrs.Bag{key: 10}); err == nil { + t.Fatalf("%s directly under meta must be rejected", key) + } + } +} + +func TestExecutionBudgetsFromOptionsValues(t *testing.T) { + valid := []struct { + raw any + want int64 + }{ + {int(-1), -1}, + {int64(512), 512}, + {uint32(7), 7}, + {float64(4096), 4096}, + {json.Number("100"), 100}, + } + for _, tc := range valid { + b, err := ExecutionBudgetsFromOptions(attrs.Bag{ProcessOptionTickBudget: tc.raw}, "test") + if err != nil || b.TickBudget != tc.want || !b.TickBudgetSet { + t.Fatalf("tick_budget %v: got %+v %v", tc.raw, b, err) + } + } + invalid := []attrs.Bag{ + {ProcessOptionTickBudget: "512"}, + {ProcessOptionTickBudget: 1.5}, + {ProcessOptionTickBudget: float64(1 << 60)}, + {ProcessOptionMaxSteps: -1}, + {ProcessOptionMaxSteps: true}, + } + for _, opts := range invalid { + if _, err := ExecutionBudgetsFromOptions(opts, "test"); err == nil { + t.Fatalf("expected %v to be rejected", opts) + } + } +} + +func TestExecutionBudgetsOverride(t *testing.T) { + entry := ExecutionBudgets{TickBudget: 100, TickBudgetSet: true, MaxSteps: 5, MaxStepsSet: true} + got := entry.Override(ExecutionBudgets{TickBudget: -1, TickBudgetSet: true}) + if got.TickBudget != -1 || got.MaxSteps != 5 || !got.MaxStepsSet { + t.Fatalf("unexpected override %+v", got) + } +} diff --git a/go.mod b/go.mod index 5171b8926..3a0362042 100644 --- a/go.mod +++ b/go.mod @@ -69,7 +69,7 @@ require ( github.com/tree-sitter/tree-sitter-php v0.25.0 github.com/tree-sitter/tree-sitter-python v0.25.0 github.com/tree-sitter/tree-sitter-typescript v0.23.2 - github.com/wippyai/go-lua v1.6.2 + github.com/wippyai/go-lua v1.6.4-0.20261002221722-a0cd38afeb7d github.com/wippyai/module-registry-proto-go v0.0.2-0.20260908140534-f6e2910c835f github.com/wippyai/tree-sitter-markdown v0.0.3 github.com/wippyai/tree-sitter-sql v0.0.4 diff --git a/go.sum b/go.sum index 8938e2de5..d783436bf 100644 --- a/go.sum +++ b/go.sum @@ -544,6 +544,8 @@ github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zd github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/wippyai/go-lua v1.6.2 h1:SJRjjW0XjDTVbFmPtjs3NHoHz1UgNdXMFJlCPg6aTQM= github.com/wippyai/go-lua v1.6.2/go.mod h1:cwD+390gJgx9cs+Zamby/lch5csqPV9RN1aDEfx/27M= +github.com/wippyai/go-lua v1.6.4-0.20261002221722-a0cd38afeb7d h1:ZJjb1KcC83wKn4ndVPXJ1r2OJoZZneIuKMPxwwXRJYE= +github.com/wippyai/go-lua v1.6.4-0.20261002221722-a0cd38afeb7d/go.mod h1:cwD+390gJgx9cs+Zamby/lch5csqPV9RN1aDEfx/27M= github.com/wippyai/module-registry-proto-go v0.0.2-0.20260908140534-f6e2910c835f h1:AoKS058roqnsHXaJ8Gww7ODz14qJqOK5QBeoALvcq6I= github.com/wippyai/module-registry-proto-go v0.0.2-0.20260908140534-f6e2910c835f/go.mod h1:p4ihYQKRQuqRVLxaH1YL0IEnkz4zKdyfVADOwvA5Tno= github.com/wippyai/tree-sitter-markdown v0.0.3 h1:u6e53+hzPSS848Xhm+I48SwtvWQHrC21wDrogdwCwuc= diff --git a/runtime/lua/component/process/manager.go b/runtime/lua/component/process/manager.go index 3d21fa0d9..0a0653e35 100644 --- a/runtime/lua/component/process/manager.go +++ b/runtime/lua/component/process/manager.go @@ -28,6 +28,7 @@ type configEntry struct { bytecode *api.BytecodeProcessConfig security *security.Config method string + budgets api.ExecutionBudgets } // Manager handles both source and bytecode Lua process components. @@ -117,7 +118,7 @@ func (m *Manager) Invalidate(ctx context.Context, ids []registry.ID) error { } } - if err := m.registerFactory(ctx, id, cfg.method, cfg.security); err != nil { + if err := m.registerFactory(ctx, id, cfg); err != nil { m.log.Error("failed to invalidate process", zap.Error(err)) errs = append(errs, err) continue @@ -156,6 +157,10 @@ func (m *Manager) addSource(ctx context.Context, entry registry.Entry) error { if err != nil { return runtimelua.NewUnpackConfigError("process", err) } + budgets, err := api.EntryExecutionBudgets(cfg.Meta) + if err != nil { + return runtimelua.NewUnpackConfigError("process", err) + } node := code.Node{ ID: entry.ID, @@ -168,9 +173,10 @@ func (m *Manager) addSource(ctx context.Context, entry registry.Entry) error { return runtimelua.NewAddNodeError("process", err) } - m.configs.Store(entry.ID, &configEntry{method: cfg.Method, source: cfg, security: cfg.Security}) + stored := &configEntry{method: cfg.Method, source: cfg, security: cfg.Security, budgets: budgets} + m.configs.Store(entry.ID, stored) - if err := m.registerFactory(ctx, entry.ID, cfg.Method, cfg.Security); err != nil { + if err := m.registerFactory(ctx, entry.ID, stored); err != nil { _ = m.code.DeleteNode(ctx, entry.ID) m.configs.Delete(entry.ID) return runtimelua.NewRegisterFactoryError(err) @@ -186,6 +192,10 @@ func (m *Manager) addBytecode(ctx context.Context, entry registry.Entry) error { if err != nil { return runtimelua.NewUnpackConfigError("process", err) } + budgets, err := api.EntryExecutionBudgets(cfg.Meta) + if err != nil { + return runtimelua.NewUnpackConfigError("process", err) + } proto, err := component.LoadAndVerifyBytecode(m.fsRegistry, cfg.FS, cfg.Path, cfg.Hash) if err != nil { @@ -202,9 +212,10 @@ func (m *Manager) addBytecode(ctx context.Context, entry registry.Entry) error { return runtimelua.NewAddNodeError("process", err) } - m.configs.Store(entry.ID, &configEntry{method: cfg.Method, bytecode: cfg, security: cfg.Security}) + stored := &configEntry{method: cfg.Method, bytecode: cfg, security: cfg.Security, budgets: budgets} + m.configs.Store(entry.ID, stored) - if err := m.registerFactory(ctx, entry.ID, cfg.Method, cfg.Security); err != nil { + if err := m.registerFactory(ctx, entry.ID, stored); err != nil { _ = m.code.DeleteNode(ctx, entry.ID) m.configs.Delete(entry.ID) return runtimelua.NewRegisterFactoryError(err) @@ -224,6 +235,10 @@ func (m *Manager) updateSource(ctx context.Context, entry registry.Entry) error if err != nil { return runtimelua.NewUnpackConfigError("process", err) } + budgets, err := api.EntryExecutionBudgets(cfg.Meta) + if err != nil { + return runtimelua.NewUnpackConfigError("process", err) + } node := code.Node{ ID: entry.ID, @@ -236,9 +251,10 @@ func (m *Manager) updateSource(ctx context.Context, entry registry.Entry) error return runtimelua.NewUpdateNodeError("process", err) } - m.configs.Store(entry.ID, &configEntry{method: cfg.Method, source: cfg, security: cfg.Security}) + stored := &configEntry{method: cfg.Method, source: cfg, security: cfg.Security, budgets: budgets} + m.configs.Store(entry.ID, stored) - if err := m.registerFactory(ctx, entry.ID, cfg.Method, cfg.Security); err != nil { + if err := m.registerFactory(ctx, entry.ID, stored); err != nil { return runtimelua.NewUpdateFactoryError(err) } @@ -252,6 +268,10 @@ func (m *Manager) updateBytecode(ctx context.Context, entry registry.Entry) erro if err != nil { return runtimelua.NewUnpackConfigError("process", err) } + budgets, err := api.EntryExecutionBudgets(cfg.Meta) + if err != nil { + return runtimelua.NewUnpackConfigError("process", err) + } proto, err := component.LoadAndVerifyBytecode(m.fsRegistry, cfg.FS, cfg.Path, cfg.Hash) if err != nil { @@ -268,9 +288,10 @@ func (m *Manager) updateBytecode(ctx context.Context, entry registry.Entry) erro return runtimelua.NewUpdateNodeError("process", err) } - m.configs.Store(entry.ID, &configEntry{method: cfg.Method, bytecode: cfg, security: cfg.Security}) + stored := &configEntry{method: cfg.Method, bytecode: cfg, security: cfg.Security, budgets: budgets} + m.configs.Store(entry.ID, stored) - if err := m.registerFactory(ctx, entry.ID, cfg.Method, cfg.Security); err != nil { + if err := m.registerFactory(ctx, entry.ID, stored); err != nil { return runtimelua.NewUpdateFactoryError(err) } @@ -279,13 +300,16 @@ func (m *Manager) updateBytecode(ctx context.Context, entry registry.Entry) erro } // registerFactory registers a process factory with the factory registry and waits for confirmation. -func (m *Manager) registerFactory(ctx context.Context, id registry.ID, method string, sec *security.Config) error { +func (m *Manager) registerFactory(ctx context.Context, id registry.ID, cfg *configEntry) error { // Create factory using ProcessFactory - factoryFn, err := m.factory.CreateFactory(id, engine.WithModules(component.ExecutableAmbientModules()...)) + factoryFn, err := m.factory.CreateFactory(id, + engine.WithModules(component.ExecutableAmbientModules()...), + engine.WithExecutionBudgets(cfg.budgets)) if err != nil { return err // Already has compile context from code.Manager } + method := cfg.method if method == "" { method = "main" } @@ -311,7 +335,7 @@ func (m *Manager) registerFactory(ctx context.Context, id registry.ID, method st Factory: factoryFn, Meta: process.Meta{ Method: method, - Security: sec, + Security: cfg.security, }, }, }) diff --git a/runtime/lua/component/process/manager_test.go b/runtime/lua/component/process/manager_test.go index 46a7a050e..0b5a0ef90 100644 --- a/runtime/lua/component/process/manager_test.go +++ b/runtime/lua/component/process/manager_test.go @@ -216,7 +216,7 @@ func TestManager_registerFactory_PreparesBeforeSend(t *testing.T) { ctx := event.WithAwaitService(ctxapi.NewRootContext(), awaitSvc) securityConfig := &security.Config{Actor: security.Actor{ID: "app.test:process"}} - err := manager.registerFactory(ctx, registry.NewID("app.test", "process"), "main", securityConfig) + err := manager.registerFactory(ctx, registry.NewID("app.test", "process"), &configEntry{method: "main", security: securityConfig}) require.NoError(t, err) assert.False(t, sendBeforePrepare, "factory register was sent before await prepare") require.Len(t, bus.events, 1) diff --git a/runtime/lua/engine/preempt_test.go b/runtime/lua/engine/preempt_test.go new file mode 100644 index 000000000..008a160dc --- /dev/null +++ b/runtime/lua/engine/preempt_test.go @@ -0,0 +1,347 @@ +// SPDX-License-Identifier: MPL-2.0 + +package engine + +import ( + "context" + "errors" + "testing" + "time" + + lua "github.com/wippyai/go-lua" + "github.com/wippyai/runtime/api/attrs" + ctxapi "github.com/wippyai/runtime/api/context" + "github.com/wippyai/runtime/api/process" + "github.com/wippyai/runtime/api/runtime" + luaapi "github.com/wippyai/runtime/api/runtime/lua" + scheduler "github.com/wippyai/runtime/system/scheduler/actor" +) + +var _ process.Preemptible = (*Process)(nil) + +// stepPreempted runs proc until it completes and returns its result value and +// the number of steps that reported StepPreempted. +func stepPreempted(t *testing.T, proc *Process, maxSteps int) (lua.LValue, int) { + t.Helper() + var output process.StepOutput + preempts := 0 + for i := 0; i < maxSteps; i++ { + output.Reset() + if err := proc.Step(nil, &output); err != nil { + t.Fatalf("step %d: %v", i, err) + } + switch output.Status() { + case process.StepPreempted: + preempts++ + case process.StepDone: + if output.Result() == nil { + t.Fatal("expected a result") + } + v, _ := output.Result().Data().(lua.LValue) + return v, preempts + case process.StepContinue: + default: + t.Fatalf("step %d: unexpected status %v", i, output.Status()) + } + } + t.Fatalf("not done after %d steps (%d preemptions)", maxSteps, preempts) + return nil, 0 +} + +// initPreemptProcess starts script with the given entry tick_budget under a +// scheduler that enables preemption; budget 0 leaves tick_budget unset. +func initPreemptProcess(t *testing.T, script string, budget int64) *Process { + t.Helper() + var budgets luaapi.ExecutionBudgets + if budget != 0 { + budgets = luaapi.ExecutionBudgets{TickBudget: budget, TickBudgetSet: true} + } + proc := mustNewProcess(t, WithScript(script, "preempt.lua"), WithProcessExecutionBudgets(budgets)) + proc.EnablePreemption() + ctx, _ := ctxapi.OpenFrameContext(context.Background()) + if err := proc.Init(ctx, "", nil); err != nil { + t.Fatalf("init: %v", err) + } + t.Cleanup(proc.Close) + return proc +} + +func TestProcessPreemptsLongRunningStep(t *testing.T) { + proc := initPreemptProcess(t, ` + local s = 0 + for i = 1, 100000 do s = s + i end + return s + `, 1000) + + v, preempts := stepPreempted(t, proc, 10000) + if lua.LVAsNumber(v) != 5000050000 { + t.Fatalf("expected 5000050000, got %v", v) + } + if preempts < 50 { + t.Fatalf("expected the loop to be split into slices, got %d preemptions", preempts) + } +} + +func TestProcessNegativeTickBudgetRunsStepToCompletion(t *testing.T) { + proc := initPreemptProcess(t, ` + local s = 0 + for i = 1, 100000 do s = s + i end + return s + `, -1) + + var output process.StepOutput + if err := proc.Step(nil, &output); err != nil { + t.Fatal(err) + } + if output.Status() != process.StepDone { + t.Fatalf("expected StepDone in one step, got %v", output.Status()) + } +} + +// The budget covers the whole step: coroutines that did not get to run +// before it ran out are deferred to the next step rather than each getting a +// fresh budget. +func TestProcessPreemptionBudgetIsPerStep(t *testing.T) { + proc := initPreemptProcess(t, ` + local done = 0 + local function work() + local s = 0 + for i = 1, 20000 do s = s + i end + done = done + 1 + end + for _ = 1, 8 do coroutine.spawn(work) end + while done < 8 do coroutine.yield() end + return done + `, 500) + + var output process.StepOutput + steps := 0 + for { + steps++ + if steps > 100000 { + t.Fatal("process did not complete") + } + output.Reset() + if err := proc.Step(nil, &output); err != nil { + t.Fatal(err) + } + if output.Status() == process.StepDone { + break + } + } + v, _ := output.Result().Data().(lua.LValue) + if lua.LVAsNumber(v) != 8 { + t.Fatalf("expected 8 finished coroutines, got %v", v) + } + // 8 coroutines * 20000 iterations need at least 320 slices of 500 ticks. + if steps < 320 { + t.Fatalf("expected per-step slicing across coroutines, completed in %d steps", steps) + } +} + +func TestProcessPreemptionKeepsSpawnedCoroutinesConsistent(t *testing.T) { + proc := initPreemptProcess(t, ` + local total = 0 + local function add(n) + local s = 0 + for i = 1, n do s = s + i end + total = total + s + end + coroutine.spawn(function() add(5000) end) + coroutine.spawn(function() add(3000) end) + add(1000) + while total ~= 12502500 + 4501500 + 500500 do coroutine.yield() end + return total + `, 300) + + v, preempts := stepPreempted(t, proc, 100000) + if lua.LVAsNumber(v) != 17504500 { + t.Fatalf("expected 17504500, got %v", v) + } + if preempts == 0 { + t.Fatal("expected preemption") + } +} + +// On a single scheduler worker, a Lua actor stuck in an endless loop is +// preempted so another actor still runs to completion. +func TestSpinningActorDoesNotStarveOtherActors(t *testing.T) { + executor := newBenchExecutorWithOptions(newMockRegistry(), + scheduler.WithWorkers(1)) + executor.Start() + defer executor.Stop() + + spinProto, err := lua.CompileString(`while true do end`, "spin.lua") + if err != nil { + t.Fatal(err) + } + quickProto, err := lua.CompileString(`local s = 0 for i = 1, 1000 do s = s + i end return s`, "quick.lua") + if err != nil { + t.Fatal(err) + } + + spinCtx, stopSpin := context.WithCancel(context.Background()) + defer stopSpin() + spinFrame, _ := ctxapi.OpenFrameContext(spinCtx) + spin := &enteredProcess{Process: mustNewProcess(t, WithProto(spinProto)), entered: make(chan struct{})} + spinDone := make(chan struct{}) + go func() { + defer close(spinDone) + _, _ = executor.Execute(spinFrame, newTestPID("spin"), spin, "", nil) + }() + select { + case <-spin.entered: + case <-time.After(5 * time.Second): + t.Fatal("spinning actor did not start") + } + + quickCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + quickFrame, _ := ctxapi.OpenFrameContext(quickCtx) + result, err := executor.Execute(quickFrame, newTestPID("quick"), &mockProcess{Process: mustNewProcess(t, WithProto(quickProto))}, "", nil) + if err != nil { + t.Fatalf("quick actor did not complete while another actor spins: %v", err) + } + if result.Error != nil { + t.Fatalf("quick actor failed: %v", result.Error) + } + if v, _ := result.Value.Data().(lua.LValue); lua.LVAsNumber(v) != 500500 { + t.Fatalf("expected 500500, got %v", result.Value.Data()) + } + + select { + case <-spinDone: + t.Fatal("spinning actor finished unexpectedly") + default: + } + stopSpin() + select { + case <-spinDone: + case <-time.After(5 * time.Second): + t.Fatal("spinning actor did not stop after cancellation") + } +} + +// enteredProcess signals when its first step begins. +type enteredProcess struct { + *Process + entered chan struct{} + started bool +} + +func (p *enteredProcess) Step(events []process.Event, out *process.StepOutput) error { + if !p.started { + p.started = true + close(p.entered) + } + return p.Process.Step(events, out) +} + +// Without a scheduler that enables preemption a step runs to completion +// whatever the tick budget. +func TestProcessNotEnabledForPreemptionRunsStepToCompletion(t *testing.T) { + proc := mustNewProcess(t, WithScript(` + local s = 0 + for i = 1, 100000 do s = s + i end + return s + `, "preempt.lua"), WithProcessExecutionBudgets(luaapi.ExecutionBudgets{TickBudget: 10, TickBudgetSet: true})) + ctx, _ := ctxapi.OpenFrameContext(context.Background()) + if err := proc.Init(ctx, "", nil); err != nil { + t.Fatal(err) + } + defer proc.Close() + var output process.StepOutput + if err := proc.Step(nil, &output); err != nil { + t.Fatal(err) + } + if output.Status() != process.StepDone { + t.Fatalf("expected StepDone in one step, got %v", output.Status()) + } +} + +func TestProcessDefaultTickBudget(t *testing.T) { + proc := initPreemptProcess(t, ` + local s = 0 + for i = 1, 100000 do s = s + i end + return s + `, 0) + _, preempts := stepPreempted(t, proc, 100000) + // 100000 iterations at the default budget of 512 ticks. + if want := 100000 / int(DefaultActorTickBudget); preempts < want { + t.Fatalf("expected at least %d preemptions at the default budget, got %d", want, preempts) + } +} + +// initWithSpawnOptions starts script with spawn options in the frame context, +// as the host does for process.with_options(...) spawns. +func initWithSpawnOptions(t *testing.T, script string, entry luaapi.ExecutionBudgets, spawn attrs.Bag) (*Process, error) { + t.Helper() + proc := mustNewProcess(t, WithScript(script, "preempt.lua"), WithProcessExecutionBudgets(entry)) + proc.EnablePreemption() + t.Cleanup(proc.Close) + ctx, fc := ctxapi.OpenFrameContext(context.Background()) + if err := fc.Set(runtime.FrameLifecycleOptionsKey, attrs.Attributes(spawn)); err != nil { + t.Fatal(err) + } + return proc, proc.Init(ctx, "", nil) +} + +func TestProcessSpawnOptionsOverrideEntryTickBudget(t *testing.T) { + script := ` + local s = 0 + for i = 1, 100000 do s = s + i end + return s + ` + entry := luaapi.ExecutionBudgets{TickBudget: 100, TickBudgetSet: true} + proc, err := initWithSpawnOptions(t, script, entry, attrs.Bag{"tick_budget": int64(-1)}) + if err != nil { + t.Fatal(err) + } + var output process.StepOutput + if err := proc.Step(nil, &output); err != nil { + t.Fatal(err) + } + if output.Status() != process.StepDone { + t.Fatalf("spawn tick_budget=-1 must disable preemption, got %v", output.Status()) + } +} + +func TestProcessRejectsInvalidSpawnOptions(t *testing.T) { + cases := []attrs.Bag{ + {"tick_budget": "fast"}, + {"tick_budget": 1.5}, + {"max_steps": int64(-1)}, + } + for _, spawn := range cases { + if _, err := initWithSpawnOptions(t, `return 1`, luaapi.ExecutionBudgets{}, spawn); err == nil { + t.Fatalf("expected %v to be rejected", spawn) + } + } +} + +func TestProcessMaxStepsFailsProcess(t *testing.T) { + script := ` + local s = 0 + for i = 1, 100000 do s = s + i end + return s + ` + entry := luaapi.ExecutionBudgets{TickBudget: 100, TickBudgetSet: true, MaxSteps: 5, MaxStepsSet: true} + proc, err := initWithSpawnOptions(t, script, entry, nil) + if err != nil { + t.Fatal(err) + } + var output process.StepOutput + for i := 0; i < 5; i++ { + output.Reset() + if err := proc.Step(nil, &output); err != nil { + t.Fatalf("step %d: %v", i, err) + } + if output.Status() != process.StepPreempted { + t.Fatalf("step %d: expected StepPreempted, got %v", i, output.Status()) + } + } + output.Reset() + if err := proc.Step(nil, &output); !errors.Is(err, process.ErrStepLimitExceeded) { + t.Fatalf("expected ErrStepLimitExceeded, got %v", err) + } +} diff --git a/runtime/lua/engine/process.go b/runtime/lua/engine/process.go index 95023ea8d..acb2bfe57 100644 --- a/runtime/lua/engine/process.go +++ b/runtime/lua/engine/process.go @@ -19,6 +19,7 @@ import ( "github.com/wippyai/runtime/api/process" "github.com/wippyai/runtime/api/registry" "github.com/wippyai/runtime/api/relay" + "github.com/wippyai/runtime/api/runtime" luaapi "github.com/wippyai/runtime/api/runtime/lua" "github.com/wippyai/runtime/api/runtime/resource" "github.com/wippyai/runtime/api/topology" @@ -31,6 +32,17 @@ var processPool = sync.Pool{ New: func() any { return nil }, } +// DefaultActorTickBudget is the tick budget of a preemptible actor whose +// tick_budget option is unset or zero. +const DefaultActorTickBudget int64 = 512 + +// WithProcessExecutionBudgets sets the process entry's execution options. +func WithProcessExecutionBudgets(budgets luaapi.ExecutionBudgets) ProcessOption { + return func(p *Process) { + p.entryBudgets = budgets + } +} + // ProcessOption configures a Process. type ProcessOption func(*Process) @@ -120,7 +132,15 @@ type Process struct { externalTasks []*Task outTasks []*Task threads []*Task - yieldSeq uint64 + // deferred holds runnable tasks that did not finish within this step's + // preemption budget; they run again in the next step. + deferred []*Task + yieldSeq uint64 + // entryBudgets are the execution options of the process entry; spawn + // options override them at Init. + entryBudgets luaapi.ExecutionBudgets + budgets luaapi.ExecutionBudgets + steps uint64 // epoch is the monotonic incarnation counter. Incremented on every // Init / clearExecution / Close drain and on Abort. Producers stamp // every SubscriptionFrame with the epoch they were registered under; @@ -130,6 +150,9 @@ type Process struct { flushingMessages bool trapLinks bool upgradable bool + // preemptive is set by schedulers that run the process again after it + // reports StepPreempted. + preemptive bool } // queuedMessage stores a message waiting to be delivered @@ -541,6 +564,13 @@ func (p *Process) Init(ctx context.Context, method string, input payload.Payload p.mainTask = nil p.pendingYields = nil + budgets, err := p.executionBudgets(ctx) + if err != nil { + return err + } + p.budgets = budgets + p.steps = 0 + // Set context for this execution p.ctx = ctx p.state.SetContext(ctx) @@ -701,6 +731,13 @@ func (p *Process) extractMethod(method string) error { // events contains yield completions and messages from the scheduler. // out is the scheduler-owned buffer where the process writes yields and status. func (p *Process) Step(events []process.Event, out *process.StepOutput) error { + if err := p.chargeStep(); err != nil { + p.clearExecution() + out.Done(nil) + return err + } + p.state.SetTickBudget(p.stepTickBudget()) + // Collect messages from events var messages []*relay.Package for _, ev := range events { @@ -798,6 +835,14 @@ func (p *Process) Step(events []process.Event, out *process.StepOutput) error { } } + // Tasks that ran out of the step's budget run again next step. + preempted := len(p.deferred) > 0 + for _, task := range p.deferred { + p.queue.Push(task) + } + clear(p.deferred) + p.deferred = p.deferred[:0] + // Check for upgrade request if p.upgradeRequest != nil { req := p.upgradeRequest @@ -846,6 +891,8 @@ func (p *Process) Step(events []process.Event, out *process.StepOutput) error { // Determine status switch { + case preempted: + out.Preempt() case yieldCount == 0 && !p.queue.IsEmpty(): out.Continue() case yieldCount == 0 && len(p.threads) > 0: @@ -871,6 +918,41 @@ func (p *Process) Step(events []process.Event, out *process.StepOutput) error { return nil } +// EnablePreemption implements process.Preemptible. Each step may then run +// the actor's tick budget of VM ticks (backward jumps, loop iterations and +// calls) before the running coroutine is preempted at a safepoint and the +// step reports StepPreempted. +func (p *Process) EnablePreemption() { + p.preemptive = true +} + +// stepTickBudget returns the VM tick budget for one step; -1 is unlimited. +func (p *Process) stepTickBudget() int64 { + if !p.preemptive { + return -1 + } + switch budget := p.budgets.TickBudget; { + case budget < 0: + return -1 + case budget == 0: + return DefaultActorTickBudget + default: + return budget + } +} + +// chargeStep counts a step against the max_steps option. +func (p *Process) chargeStep() error { + if p.budgets.MaxSteps == 0 { + return nil + } + p.steps++ + if p.steps > p.budgets.MaxSteps { + return process.ErrStepLimitExceeded + } + return nil +} + // State returns the underlying Lua state. func (p *Process) State() *lua.LState { return p.state @@ -1709,7 +1791,11 @@ func (p *Process) vmStep(tasks ...*Task) ([]*Task, error) { continue } - if task.State != lua.ResumeYield { + if task.State != lua.ResumeYield && task.State != lua.ResumePreempted { + continue + } + if p.state.TickBudget() == 0 { + p.deferred = append(p.deferred, task) continue } @@ -1760,6 +1846,8 @@ func (p *Process) vmStep(tasks ...*Task) ([]*Task, error) { return nil, nil } p.removeTask(task) + case lua.ResumePreempted: + p.deferred = append(p.deferred, task) case lua.ResumeError: p.removeTask(task) } @@ -1805,6 +1893,8 @@ func (p *Process) killAllThreads() { } p.threads = p.threads[:0] p.queue.Drain() + clear(p.deferred) + p.deferred = p.deferred[:0] } // handleSpawnRequest checks if yielded values contain a SpawnRequest and handles it. @@ -1894,6 +1984,8 @@ func (p *Process) Close() { p.yieldBuf = p.yieldBuf[:0] p.externalTasks = p.externalTasks[:0] p.outTasks = p.outTasks[:0] + clear(p.deferred) + p.deferred = p.deferred[:0] p.clearMessageQueue() // Clear all references @@ -1924,11 +2016,30 @@ func (p *Process) Close() { p.upgradable = false p.pendingOutdated = nil p.linkDownError = nil + p.entryBudgets = luaapi.ExecutionBudgets{} + p.budgets = luaapi.ExecutionBudgets{} + p.steps = 0 + p.preemptive = false // Return to pool processPool.Put(p) } +// executionBudgets resolves the execution options for this execution: the +// entry's options, overridden by the spawn options in the frame context. +func (p *Process) executionBudgets(ctx context.Context) (luaapi.ExecutionBudgets, error) { + budgets := p.entryBudgets + options, ok := runtime.GetFrameLifecycleOptions(ctx).(attrs.Attributes) + if !ok || options == nil { + return budgets, nil + } + spawn, err := luaapi.ExecutionBudgetsFromOptions(options, "process spawn options") + if err != nil { + return budgets, err + } + return budgets.Override(spawn), nil +} + // SyncExecute runs the script directly without coroutines or scheduler. // This is a fast path for simple synchronous calls that don't need yields. // The Lua state must be initialized via Start first. @@ -2019,6 +2130,8 @@ func (p *Process) clearExecution() { // Drain queue p.queue.Drain() + clear(p.deferred) + p.deferred = p.deferred[:0] // Clear main task reference and result p.mainTask = nil diff --git a/runtime/lua/engine/process_factory.go b/runtime/lua/engine/process_factory.go index a66120843..a57b3cf37 100644 --- a/runtime/lua/engine/process_factory.go +++ b/runtime/lua/engine/process_factory.go @@ -49,6 +49,7 @@ type processConfig struct { excludeModules []string extraModules []*luaapi.ModuleDef buildMode code.AccessMode + budgets luaapi.ExecutionBudgets } func newProcessConfig() *processConfig { @@ -107,6 +108,14 @@ func WithModules(mods ...*luaapi.ModuleDef) FactoryOption { } } +// WithExecutionBudgets sets the execution options (tick_budget, max_steps) +// of the processes the factory creates. +func WithExecutionBudgets(budgets luaapi.ExecutionBudgets) FactoryOption { + return func(c *processConfig) { + c.budgets = budgets + } +} + // WithFilter sets a custom filter function. // Return (true, nil) to include, (false, nil) to exclude, (false, err) to fail. func WithFilter(fn func(name string, classes []string) (bool, error)) FactoryOption { @@ -152,6 +161,7 @@ func (f *ProcessFactory) CreateFactory(id registry.ID, opts ...FactoryOption) (p factoryCfg := FactoryConfig{ Proto: compiled.Main, ModuleBinders: binders, + Budgets: cfg.budgets, } factory := NewFactory(factoryCfg) @@ -372,6 +382,7 @@ type FactoryConfig struct { Script string ScriptName string ModuleBinders []ModuleBinder + Budgets luaapi.ExecutionBudgets } // Factory creates Lua processes with shared configuration. @@ -382,6 +393,7 @@ type Factory struct { script string scriptName string moduleBinders []ModuleBinder + budgets luaapi.ExecutionBudgets } // NewFactory creates a ProcessFactory for Lua processes. @@ -393,6 +405,7 @@ func NewFactory(cfg FactoryConfig) process.FactoryFunc { scriptName: cfg.ScriptName, moduleBinders: cfg.ModuleBinders, stateOpts: cfg.StateOptions, + budgets: cfg.Budgets, } return f.Create } @@ -405,11 +418,12 @@ func (f *Factory) Create() (process.Process, error) { } proc := &Process{ - threads: make([]*Task, 0, 4), - queue: NewTaskQueue(), - yieldBuf: make([]*Task, 0, 4), - factory: f, - state: state, + threads: make([]*Task, 0, 4), + queue: NewTaskQueue(), + yieldBuf: make([]*Task, 0, 4), + factory: f, + state: state, + entryBudgets: f.budgets, } if f.proto != nil { diff --git a/runtime/lua/modules/process/context.go b/runtime/lua/modules/process/context.go index 3da067899..d8fd31638 100644 --- a/runtime/lua/modules/process/context.go +++ b/runtime/lua/modules/process/context.go @@ -14,6 +14,7 @@ import ( "github.com/wippyai/runtime/api/registry" "github.com/wippyai/runtime/api/relay" "github.com/wippyai/runtime/api/runtime" + luaapi "github.com/wippyai/runtime/api/runtime/lua" secapi "github.com/wippyai/runtime/api/security" "github.com/wippyai/runtime/runtime/lua/engine/value" "github.com/wippyai/runtime/runtime/security" @@ -86,6 +87,11 @@ func parseSpawnerOptions(l *lua.LState, idx int) attrs.Bag { options.Set(string(key), value.ToGoAny(v)) }) + if _, err := luaapi.ExecutionBudgetsFromOptions(options, "process spawn options"); err != nil { + l.ArgError(idx, err.Error()) + return nil + } + if net := options.GetString(netapi.OptionKeyNetwork, ""); net != "" { if !security.IsAllowed(l.Context(), "network.select", net, nil) { l.RaiseError("not allowed: network %s", net) diff --git a/runtime/lua/modules/process/spawner_security_test.go b/runtime/lua/modules/process/spawner_security_test.go index 000991e36..8f7fa4bc9 100644 --- a/runtime/lua/modules/process/spawner_security_test.go +++ b/runtime/lua/modules/process/spawner_security_test.go @@ -87,3 +87,15 @@ func TestSpawnerContextPermissionStillRequired(t *testing.T) { require.ErrorContains(t, l.DoString(`process.with_options({})`), "custom options") require.ErrorContains(t, l.DoString(`process.with_context({})`), "custom context") } + +func TestSpawnerOptionsValidateExecutionBudgets(t *testing.T) { + l, _ := spawnerSecurityLua(t, &spawnerTestPolicy{contextAllowed: true}) + require.NoError(t, l.DoString(`result = process.with_options({tick_budget = 2048, max_steps = 10})`)) + spawner := l.GetGlobal("result").(*lua.LUserData).Value.(*Spawner) + budget, _ := spawner.options.Get("tick_budget") + require.EqualValues(t, 2048, budget) + + require.ErrorContains(t, l.DoString(`process.with_options({tick_budget = "fast"})`), `"tick_budget" must be an integer`) + require.ErrorContains(t, l.DoString(`process.with_options({tick_budget = 1.5})`), `"tick_budget" must be an integer`) + require.ErrorContains(t, l.DoString(`process.with_options({max_steps = -1})`), `"max_steps" must be a non-negative integer`) +} diff --git a/system/scheduler/actor/preempt_test.go b/system/scheduler/actor/preempt_test.go new file mode 100644 index 000000000..b0d9776a2 --- /dev/null +++ b/system/scheduler/actor/preempt_test.go @@ -0,0 +1,198 @@ +// SPDX-License-Identifier: MPL-2.0 + +package actor + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/wippyai/runtime/api/payload" + pidapi "github.com/wippyai/runtime/api/pid" + "github.com/wippyai/runtime/api/process" + "github.com/wippyai/runtime/api/relay" + "github.com/wippyai/runtime/api/runtime" + "github.com/wippyai/runtime/system/scheduler" +) + +// spinProcess never finishes on its own: every step uses up its budget. +type spinProcess struct { + enabled atomic.Bool + release atomic.Bool + steps atomic.Int64 +} + +func (*spinProcess) Init(context.Context, string, payload.Payloads) error { return nil } +func (p *spinProcess) EnablePreemption() { p.enabled.Store(true) } +func (p *spinProcess) Step(_ []process.Event, out *process.StepOutput) error { + p.steps.Add(1) + if p.release.Load() { + out.Done(nil) + return nil + } + out.Preempt() + return nil +} +func (*spinProcess) Send(*relay.Package) error { return nil } +func (*spinProcess) Close() {} + +// pingProcess yields rounds times and completes. +type pingProcess struct { + rounds int + done int +} + +func (*pingProcess) Init(context.Context, string, payload.Payloads) error { return nil } +func (p *pingProcess) Step(events []process.Event, out *process.StepOutput) error { + p.done += len(events) + if p.done >= p.rounds { + out.Done(nil) + return nil + } + if len(events) > 0 || p.done == 0 { + out.Yield(YieldCmd{}, uint64(p.done+1)) + } + out.WaitForYields() + return nil +} +func (*pingProcess) Send(*relay.Package) error { return nil } +func (*pingProcess) Close() {} + +// preemptWithPendingYieldProcess dispatches a long sleep and reports +// preemption in the same step; it must run again without the sleep finishing. +type preemptWithPendingYieldProcess struct { + steps atomic.Int64 +} + +func (*preemptWithPendingYieldProcess) Init(context.Context, string, payload.Payloads) error { + return nil +} +func (p *preemptWithPendingYieldProcess) Step(_ []process.Event, out *process.StepOutput) error { + if p.steps.Add(1) == 1 { + out.Yield(SleepCmd{Duration: time.Hour}, 1) + out.Preempt() + return nil + } + out.Done(nil) + return nil +} +func (*preemptWithPendingYieldProcess) Send(*relay.Package) error { return nil } +func (*preemptWithPendingYieldProcess) Close() {} + +func newPreemptTestScheduler(workers int, lc *testLifecycle, opts ...Option) *Scheduler { + registry := scheduler.NewRegistry() + registry.Register(CmdComplete, CompleteHandler()) + registry.Register(CmdYield, YieldHandler()) + registry.Register(CmdSleep, SleepHandler()) + return NewScheduler(registry, append([]Option{WithWorkers(workers), WithLifecycle(lc)}, opts...)...) +} + +func TestSchedulerEnablesPreemptionForPreemptibleProcesses(t *testing.T) { + sched := newPreemptTestScheduler(1, &testLifecycle{}) + sched.Start() + defer testStopScheduler(sched) + p := &spinProcess{} + p.release.Store(true) + if _, err := sched.Submit(context.Background(), pidapi.PID{UniqID: "a"}, p, "", nil); err != nil { + t.Fatal(err) + } + if !p.enabled.Load() { + t.Fatal("expected preemption to be enabled at submit") + } + q := &spinProcess{} + q.release.Store(true) + if _, err := sched.CreateProcessor(context.Background(), pidapi.PID{UniqID: "b"}, q); err != nil { + t.Fatal(err) + } + if !q.enabled.Load() { + t.Fatal("expected preemption to be enabled for created processors") + } +} + +// A preempted process goes behind other ready work: on a single worker a +// process that keeps yielding completes while a spinning process runs. +func TestPreemptedProcessYieldsWorkerToReadyProcesses(t *testing.T) { + var pingDone atomic.Bool + lc := &testLifecycle{ + onComplete: func(_ context.Context, pid pidapi.PID, _ *runtime.Result) { + if pid.UniqID == "ping" { + pingDone.Store(true) + } + }, + } + sched := newPreemptTestScheduler(1, lc) + sched.Start() + defer testStopScheduler(sched) + + spin := &spinProcess{} + defer spin.release.Store(true) + if _, err := sched.Submit(context.Background(), pidapi.PID{UniqID: "spin"}, spin, "", nil); err != nil { + t.Fatal(err) + } + const rounds = 50 + if _, err := sched.Submit(context.Background(), pidapi.PID{UniqID: "ping"}, &pingProcess{rounds: rounds}, "", nil); err != nil { + t.Fatal(err) + } + + deadline := time.Now().Add(5 * time.Second) + for !pingDone.Load() && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if !pingDone.Load() { + t.Fatalf("ready process starved by preempted process (%d spin steps)", spin.steps.Load()) + } +} + +// After a preempted step the worker serves its other ready work before the +// preempted process runs again. +func TestPreemptedProcessIsQueuedBehindLocalReadyWork(t *testing.T) { + sched := newPreemptTestScheduler(1, &testLifecycle{}) + spin := &spinProcess{} + spinProc, err := sched.Submit(context.Background(), pidapi.PID{UniqID: "spin"}, spin, "", nil) + if err != nil { + t.Fatal(err) + } + other := &spinProcess{} + otherProc, err := sched.Submit(context.Background(), pidapi.PID{UniqID: "other"}, other, "", nil) + if err != nil { + t.Fatal(err) + } + + w := sched.workerSnapshot()[0] + if got := w.findWork(); got != spinProc { + t.Fatalf("expected the first submission, got %p", got) + } + w.executeOne(spinProc) + if spin.steps.Load() != 1 { + t.Fatalf("expected one spin step, got %d", spin.steps.Load()) + } + if got := w.findWork(); got != otherProc { + t.Fatalf("expected ready work before the preempted process, got %p (spin %p)", got, spinProc) + } + if got := w.findWork(); got != spinProc { + t.Fatalf("expected the preempted process to be runnable again, got %p", got) + } +} + +func TestPreemptedProcessRunsAgainWithPendingYields(t *testing.T) { + var done atomic.Bool + lc := &testLifecycle{ + onComplete: func(context.Context, pidapi.PID, *runtime.Result) { done.Store(true) }, + } + sched := newPreemptTestScheduler(1, lc) + sched.Start() + defer testStopScheduler(sched) + + p := &preemptWithPendingYieldProcess{} + if _, err := sched.Submit(context.Background(), testPID(), p, "", nil); err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(2 * time.Second) + for !done.Load() && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if !done.Load() { + t.Fatalf("preempted process with a pending yield was not run again (%d steps)", p.steps.Load()) + } +} diff --git a/system/scheduler/actor/scheduler.go b/system/scheduler/actor/scheduler.go index c35c22a2d..0337ac6d1 100644 --- a/system/scheduler/actor/scheduler.go +++ b/system/scheduler/actor/scheduler.go @@ -235,6 +235,23 @@ func (s *Scheduler) wakeAny() { } } +// enablePreemption lets a preemptible process suspend long steps: the +// scheduler runs it again after StepPreempted. +func (s *Scheduler) enablePreemption(p process.Process) { + if pp, ok := p.(process.Preemptible); ok { + pp.EnablePreemption() + } +} + +// requeuePreempted makes a preempted processor runnable behind other ready +// work. The global queue is FIFO and shared, so an idle worker can pick it up +// while this worker serves its local and injected work first. +func (s *Scheduler) requeuePreempted(proc *Processor) { + proc.lastWorker.Store(noWorkerAffinity) + s.global.Push(proc) + s.wakeAny() +} + func (s *Scheduler) wakeAll() { for _, w := range s.workerSnapshot() { _ = w.signal() @@ -314,6 +331,7 @@ func (s *Scheduler) Submit(ctx context.Context, pid pid.PID, p process.Process, procCtx, cancel = context.WithCancel(ctx) } + s.enablePreemption(p) if err := p.Init(procCtx, method, input); err != nil { cancel() return nil, err @@ -449,6 +467,8 @@ func (s *Scheduler) CreateProcessor(ctx context.Context, pid pid.PID, p process. return nil, process.ErrMaxProcessesExceeded } + s.enablePreemption(p) + // Wrap context with cancel for Terminate support procCtx, cancel := context.WithCancel(ctx) diff --git a/system/scheduler/actor/worker.go b/system/scheduler/actor/worker.go index 24a385211..ecf7e74e9 100644 --- a/system/scheduler/actor/worker.go +++ b/system/scheduler/actor/worker.go @@ -398,7 +398,7 @@ func (w *Worker) executeOne(proc *Processor) { if status != process.StepDone { yields := proc.output.Yields() if len(yields) > 0 { - w.dispatchYields(proc.ctx, proc, yields) + w.dispatchYields(proc.ctx, proc, yields, status == process.StepPreempted) return } } @@ -420,6 +420,12 @@ func (w *Worker) executeOne(proc *Processor) { // No wake needed since we're the active worker. w.local.Push(proc) + case process.StepPreempted: + if !proc.casState(StateRunning, StateReady) { + return + } + w.scheduler.requeuePreempted(proc) + case process.StepYield: if !proc.casState(StateRunning, StateBlocked) { return @@ -555,7 +561,9 @@ func (w *Worker) executeOne(proc *Processor) { // dispatchYields sends all yields to handlers. // Processor state is StateRunning during this call. // CompleteYield sets wakeup flag instead of re-queueing while Running. -func (w *Worker) dispatchYields(ctx context.Context, proc *Processor, yields []process.Yield) { +// A preempted process stays runnable and is requeued; otherwise it blocks +// until a completion wakes it. +func (w *Worker) dispatchYields(ctx context.Context, proc *Processor, yields []process.Yield, preempted bool) { for _, y := range yields { handler := w.scheduler.getHandler(y.Cmd) if handler == nil { @@ -575,6 +583,13 @@ func (w *Worker) dispatchYields(ctx context.Context, proc *Processor, yields []p } } + if preempted { + if proc.casState(StateRunning, StateReady) { + w.scheduler.requeuePreempted(proc) + } + return + } + // Atomically transition to final state. // If wakeup was set by CompleteYield, state becomes Ready - re-queue. // Otherwise, state becomes Blocked. From 464097dafbb123a24595d2a88d2ba0d1352f6f15 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Fri, 2 Oct 2026 20:13:03 -0400 Subject: [PATCH 02/50] fix(actor): keep preemption and execution limits across process upgrade Upgrade replacements are admitted for preemption like any other process, inherit the spawn lifecycle options of the actor, and continue its step count so max_steps bounds the actor over its whole lifetime. A preempted process goes straight back to its worker when nothing else is ready and wakes another worker only when one is parked. --- api/process/preempt.go | 19 ++- api/runtime/context.go | 10 ++ system/scheduler/actor/inject.go | 6 + system/scheduler/actor/preempt_bench_test.go | 114 +++++++++++++ system/scheduler/actor/preempt_test.go | 161 +++++++++++++++++++ system/scheduler/actor/scheduler.go | 10 +- system/scheduler/actor/worker.go | 52 +++++- 7 files changed, 358 insertions(+), 14 deletions(-) create mode 100644 system/scheduler/actor/preempt_bench_test.go diff --git a/api/process/preempt.go b/api/process/preempt.go index 026471618..1c785058e 100644 --- a/api/process/preempt.go +++ b/api/process/preempt.go @@ -4,9 +4,22 @@ package process // Preemptible is an optional interface for processes that can suspend a // running step at a safepoint once they have used up their execution slice. -// A scheduler calls EnablePreemption when it runs the process again after -// Step reports StepPreempted; a process that was never enabled runs each step -// to a blocking point or completion. +// A scheduler calls EnablePreemption once, when it admits the process, and +// the opt-in persists for the process's lifetime. It then runs the process +// again after Step reports StepPreempted; a process that was never enabled +// runs each step to a blocking point or completion. type Preemptible interface { EnablePreemption() } + +// StepAccounted is an optional interface for processes that count their +// scheduler steps against a limit. A scheduler carries the count from a +// process to its upgrade replacement so the limit covers the whole lifetime +// of the actor rather than each incarnation. +type StepAccounted interface { + // StepsUsed returns the number of steps taken so far. + StepsUsed() uint64 + // ResumeStepCount sets the number of steps already taken. It is called + // after Init, before the first Step. + ResumeStepCount(used uint64) +} diff --git a/api/runtime/context.go b/api/runtime/context.go index 30ced5318..817735556 100644 --- a/api/runtime/context.go +++ b/api/runtime/context.go @@ -73,6 +73,16 @@ func GetFramePID(ctx context.Context) (pid.PID, bool) { return pid.PID{}, false } +// SetFrameLifecycleOptions sets the lifecycle options (attrs.Attributes) in +// the FrameContext. Returns error if no frame context or frame is sealed. +func SetFrameLifecycleOptions(ctx context.Context, opts any) error { + fc := ctxapi.FrameFromContext(ctx) + if fc == nil { + return ctxapi.ErrNoFrameContext + } + return fc.Set(FrameLifecycleOptionsKey, opts) +} + // GetFrameLifecycleOptions retrieves lifecycle options from the FrameContext. // Returns nil if not found. func GetFrameLifecycleOptions(ctx context.Context) any { diff --git a/system/scheduler/actor/inject.go b/system/scheduler/actor/inject.go index 1fe537dd3..d44f07582 100644 --- a/system/scheduler/actor/inject.go +++ b/system/scheduler/actor/inject.go @@ -70,6 +70,12 @@ func (q *InjectQueue) Pop() *Processor { return proc } +// IsEmpty reports whether no processor is queued. Only safe to call from the +// single consumer. +func (q *InjectQueue) IsEmpty() bool { + return q.tail.next.Load() == nil +} + // Drain pops all available processors into dst, returns count. // Useful for batch processing. func (q *InjectQueue) Drain(dst []*Processor) int { diff --git a/system/scheduler/actor/preempt_bench_test.go b/system/scheduler/actor/preempt_bench_test.go new file mode 100644 index 000000000..d52ca3949 --- /dev/null +++ b/system/scheduler/actor/preempt_bench_test.go @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: MPL-2.0 + +package actor + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "testing" + + "github.com/wippyai/runtime/api/payload" + pidapi "github.com/wippyai/runtime/api/pid" + "github.com/wippyai/runtime/api/process" + "github.com/wippyai/runtime/api/relay" + "github.com/wippyai/runtime/api/runtime" +) + +var benchSink atomic.Uint64 + +// sliceProcess burns a fixed amount of work per step and reports preemption +// until the shared slice budget is used up. +type sliceProcess struct { + slices *atomic.Int64 + work int +} + +func (*sliceProcess) Init(context.Context, string, payload.Payloads) error { return nil } +func (p *sliceProcess) EnablePreemption() {} +func (p *sliceProcess) Step(_ []process.Event, out *process.StepOutput) error { + if p.slices.Add(-1) < 0 { + out.Done(nil) + return nil + } + var acc uint64 + for i := 0; i < p.work; i++ { + acc += uint64(i) * 2654435761 + } + benchSink.Add(acc & 1) + out.Preempt() + return nil +} +func (*sliceProcess) Send(*relay.Package) error { return nil } +func (*sliceProcess) Close() {} + +// BenchmarkPreemptedSlices measures scheduler cost per preempted step with +// one or two spinning actors per worker; ns/op is wall time per slice. +func BenchmarkPreemptedSlices(b *testing.B) { + for _, workers := range []int{1, 2, 4, 8, 16} { + for _, perWorker := range []int{1, 2} { + b.Run(fmt.Sprintf("workers=%d/actors=%dx", workers, perWorker), func(b *testing.B) { + const work = 500 + var wg sync.WaitGroup + lc := &testLifecycle{ + onComplete: func(context.Context, pidapi.PID, *runtime.Result) { wg.Done() }, + } + sched := newPreemptTestScheduler(workers, lc) + sched.Start() + defer testStopScheduler(sched) + + var slices atomic.Int64 + slices.Store(int64(b.N)) + actors := workers * perWorker + wg.Add(actors) + b.ResetTimer() + for i := 0; i < actors; i++ { + p := &sliceProcess{slices: &slices, work: work} + if _, err := sched.Submit(context.Background(), pidapi.PID{UniqID: fmt.Sprintf("spin-%d", i)}, p, "", nil); err != nil { + b.Fatal(err) + } + } + wg.Wait() + }) + } + } +} + +// BenchmarkPreemptedRequeue drives workers by hand, one goroutine per worker +// and without parking, to measure the dispatch and requeue cost of a +// preempted step; ns/op is per slice across all workers. +func BenchmarkPreemptedRequeue(b *testing.B) { + for _, workers := range []int{1, 4, 16} { + for _, perWorker := range []int{1, 2} { + b.Run(fmt.Sprintf("workers=%d/actors=%dx", workers, perWorker), func(b *testing.B) { + sched := newPreemptTestScheduler(workers, &testLifecycle{}) + var slices atomic.Int64 + slices.Store(int64(b.N) + int64(workers*perWorker)) + for i := 0; i < workers*perWorker; i++ { + p := &sliceProcess{slices: &slices} + if _, err := sched.Submit(context.Background(), pidapi.PID{UniqID: fmt.Sprintf("spin-%d", i)}, p, "", nil); err != nil { + b.Fatal(err) + } + } + snapshot := sched.workerSnapshot() + var next atomic.Int32 + b.ResetTimer() + var wg sync.WaitGroup + for g := 0; g < workers; g++ { + wg.Add(1) + go func() { + defer wg.Done() + w := snapshot[int(next.Add(1))-1] + for slices.Load() > 0 { + if proc := w.findWork(); proc != nil { + w.executeOne(proc) + } + } + }() + } + wg.Wait() + }) + } + } +} diff --git a/system/scheduler/actor/preempt_test.go b/system/scheduler/actor/preempt_test.go index b0d9776a2..5a55af72b 100644 --- a/system/scheduler/actor/preempt_test.go +++ b/system/scheduler/actor/preempt_test.go @@ -8,9 +8,12 @@ import ( "testing" "time" + "github.com/wippyai/runtime/api/attrs" + ctxapi "github.com/wippyai/runtime/api/context" "github.com/wippyai/runtime/api/payload" pidapi "github.com/wippyai/runtime/api/pid" "github.com/wippyai/runtime/api/process" + "github.com/wippyai/runtime/api/registry" "github.com/wippyai/runtime/api/relay" "github.com/wippyai/runtime/api/runtime" "github.com/wippyai/runtime/system/scheduler" @@ -196,3 +199,161 @@ func TestPreemptedProcessRunsAgainWithPendingYields(t *testing.T) { t.Fatalf("preempted process with a pending yield was not run again (%d steps)", p.steps.Load()) } } + +// upgradeSource requests an upgrade on its first step and reports a fixed +// step count to the scheduler. +type upgradeSource struct { + UpgradeProcess + used uint64 +} + +func (p *upgradeSource) StepsUsed() uint64 { return p.used } +func (p *upgradeSource) ResumeStepCount(n uint64) { p.used = n } + +// upgradeTarget records how the scheduler prepared it as an upgrade replacement. +type upgradeTarget struct { + stop chan struct{} + options atomic.Value + resumed atomic.Uint64 + enabled atomic.Bool + initialized atomic.Bool + spinning bool +} + +func (p *upgradeTarget) Init(ctx context.Context, _ string, _ payload.Payloads) error { + p.initialized.Store(true) + if opts := runtime.GetFrameLifecycleOptions(ctx); opts != nil { + p.options.Store(opts) + } + return nil +} +func (p *upgradeTarget) EnablePreemption() { p.enabled.Store(true) } +func (p *upgradeTarget) StepsUsed() uint64 { return p.resumed.Load() } +func (p *upgradeTarget) ResumeStepCount(used uint64) { + p.resumed.Store(used) +} +func (p *upgradeTarget) Step(_ []process.Event, out *process.StepOutput) error { + if p.spinning { + if p.enabled.Load() { + out.Preempt() + return nil + } + // Without preemption the step never ends on its own. + select { + case <-p.stop: + case <-time.After(10 * time.Second): + } + } + out.Done(nil) + return nil +} +func (*upgradeTarget) Send(*relay.Package) error { return nil } +func (*upgradeTarget) Close() {} + +// upgradeInto submits an upgrade source whose replacement is target and waits +// until the replacement is initialized. +func upgradeInto(ctx context.Context, t *testing.T, sched *Scheduler, source process.Process, target *upgradeTarget) { + t.Helper() + appCtx := ctxapi.WithAppContext(ctx, ctxapi.NewAppContext()) + process.WithFactory(appCtx, &mockFactory{ + createFunc: func(registry.ID) (process.Process, *process.Meta, error) { + return target, &process.Meta{Method: "main"}, nil + }, + }) + frameCtx, fc := ctxapi.OpenFrameContext(appCtx) + fc.Seal() + if _, err := sched.Submit(frameCtx, pidapi.PID{UniqID: "upgrader"}, source, "", nil); err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(5 * time.Second) + for !target.initialized.Load() && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if !target.initialized.Load() { + t.Fatal("upgrade replacement was not initialized") + } +} + +func TestUpgradeReplacementIsPreemptionEnabled(t *testing.T) { + sched := newPreemptTestScheduler(1, &testLifecycle{}) + sched.Start() + defer testStopScheduler(sched) + + target := &upgradeTarget{} + source := &upgradeSource{UpgradeProcess: UpgradeProcess{upgradeReq: &process.UpgradeRequest{Source: registry.NewID("app", "next")}}} + upgradeInto(context.Background(), t, sched, source, target) + if !target.enabled.Load() { + t.Fatal("expected preemption to be enabled on the upgrade replacement") + } +} + +// An actor that upgrades into a spinning definition still yields its worker. +func TestUpgradeIntoSpinningActorYieldsWorker(t *testing.T) { + var pingDone atomic.Bool + lc := &testLifecycle{ + onComplete: func(_ context.Context, pid pidapi.PID, _ *runtime.Result) { + if pid.UniqID == "ping" { + pingDone.Store(true) + } + }, + } + sched := newPreemptTestScheduler(1, lc) + sched.Start() + defer testStopScheduler(sched) + + target := &upgradeTarget{spinning: true, stop: make(chan struct{})} + defer close(target.stop) + source := &upgradeSource{UpgradeProcess: UpgradeProcess{upgradeReq: &process.UpgradeRequest{Source: registry.NewID("app", "next")}}} + upgradeInto(context.Background(), t, sched, source, target) + + if _, err := sched.Submit(context.Background(), pidapi.PID{UniqID: "ping"}, &pingProcess{rounds: 20}, "", nil); err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(5 * time.Second) + for !pingDone.Load() && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if !pingDone.Load() { + t.Fatal("ready process starved by the upgraded actor") + } +} + +func TestUpgradeKeepsSpawnLifecycleOptions(t *testing.T) { + sched := newPreemptTestScheduler(1, &testLifecycle{}) + sched.Start() + defer testStopScheduler(sched) + + spawn := attrs.Bag{"max_steps": int64(7)} + ctx, fc := ctxapi.OpenFrameContext(context.Background()) + if err := fc.Set(runtime.FrameLifecycleOptionsKey, attrs.Attributes(spawn)); err != nil { + t.Fatal(err) + } + target := &upgradeTarget{} + source := &upgradeSource{UpgradeProcess: UpgradeProcess{upgradeReq: &process.UpgradeRequest{Source: registry.NewID("app", "next")}}} + upgradeInto(ctx, t, sched, source, target) + + got, _ := target.options.Load().(attrs.Attributes) + if got == nil || got.GetInt("max_steps", 0) != 7 { + t.Fatalf("expected spawn options on the upgrade frame, got %v", got) + } +} + +func TestUpgradeCarriesStepCount(t *testing.T) { + sched := newPreemptTestScheduler(1, &testLifecycle{}) + sched.Start() + defer testStopScheduler(sched) + + target := &upgradeTarget{} + source := &upgradeSource{ + UpgradeProcess: UpgradeProcess{upgradeReq: &process.UpgradeRequest{Source: registry.NewID("app", "next")}}, + used: 41, + } + upgradeInto(context.Background(), t, sched, source, target) + deadline := time.Now().Add(5 * time.Second) + for target.resumed.Load() == 0 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if got := target.resumed.Load(); got != 41 { + t.Fatalf("expected 41 steps carried to the replacement, got %d", got) + } +} diff --git a/system/scheduler/actor/scheduler.go b/system/scheduler/actor/scheduler.go index 0337ac6d1..68053bf6f 100644 --- a/system/scheduler/actor/scheduler.go +++ b/system/scheduler/actor/scheduler.go @@ -114,6 +114,7 @@ type Scheduler struct { retiredStolen atomic.Uint64 queueSize int nextID atomic.Uint64 + parked atomic.Int32 phase atomic.Uint32 collectStats atomic.Bool started bool @@ -243,15 +244,6 @@ func (s *Scheduler) enablePreemption(p process.Process) { } } -// requeuePreempted makes a preempted processor runnable behind other ready -// work. The global queue is FIFO and shared, so an idle worker can pick it up -// while this worker serves its local and injected work first. -func (s *Scheduler) requeuePreempted(proc *Processor) { - proc.lastWorker.Store(noWorkerAffinity) - s.global.Push(proc) - s.wakeAny() -} - func (s *Scheduler) wakeAll() { for _, w := range s.workerSnapshot() { _ = w.signal() diff --git a/system/scheduler/actor/worker.go b/system/scheduler/actor/worker.go index ecf7e74e9..7f8b5bf2e 100644 --- a/system/scheduler/actor/worker.go +++ b/system/scheduler/actor/worker.go @@ -163,15 +163,40 @@ func (w *Worker) handoff(proc *Processor) { w.scheduler.global.Push(proc) } +// requeuePreempted makes a preempted processor runnable behind other ready +// work. When this worker has nothing else to run the processor continues on +// the local deque like StepContinue; otherwise it joins the global FIFO, where +// any worker can pick it up after the work already queued. The wakeup is only +// needed when some worker is parked: this worker looks for work again right +// after the step. +func (w *Worker) requeuePreempted(proc *Processor) { + s := w.scheduler + if w.local.IsEmpty() && w.inject.IsEmpty() && s.global.Len() == 0 { + w.local.Push(proc) + return + } + proc.lastWorker.Store(noWorkerAffinity) + s.global.Push(proc) + if s.parked.Load() > 0 { + s.wakeAny() + } +} + func (w *Worker) park() { s := w.scheduler + // A worker counts as parked from before its last work check until it + // stops waiting, so a producer that publishes work and then reads the + // count either sees the worker or the worker sees the work. + s.parked.Add(1) w.parkMu.Lock() for { if w.retiring.Load() { + s.parked.Add(-1) w.parkMu.Unlock() return } if proc := w.findWork(); proc != nil { + s.parked.Add(-1) shouldWake := s.global.Len() > 0 w.parkMu.Unlock() if shouldWake { @@ -182,6 +207,7 @@ func (w *Worker) park() { return } if s.phase.Load() == phaseStoppingWorkers { + s.parked.Add(-1) w.parkMu.Unlock() return } @@ -424,7 +450,7 @@ func (w *Worker) executeOne(proc *Processor) { if !proc.casState(StateRunning, StateReady) { return } - w.scheduler.requeuePreempted(proc) + w.requeuePreempted(proc) case process.StepYield: if !proc.casState(StateRunning, StateBlocked) { @@ -497,11 +523,17 @@ func (w *Worker) executeOne(proc *Processor) { return } + var stepsUsed uint64 + if counter, ok := proc.Process.(process.StepAccounted); ok { + stepsUsed = counter.StepsUsed() + } + // Close old process proc.Process.Close() // Swap proc.Process = newProc + w.scheduler.enablePreemption(newProc) // Init new process method := "main" @@ -523,6 +555,19 @@ func (w *Worker) executeOne(proc *Processor) { w.scheduler.complete(proc, nil, fmt.Errorf("upgrade: set source failed: %w", err)) return } + // Spawn options (execution limits, parent) describe the actor, not its + // current definition, so the replacement frame carries them over. + if opts := runtime.GetFrameLifecycleOptions(proc.ctx); opts != nil { + if err := runtime.SetFrameLifecycleOptions(upgradeCtx, opts); err != nil { + proc.Process.Close() + if !proc.casState(StateRunning, StateComplete) { + return + } + proc.queue.Close() + w.scheduler.complete(proc, nil, fmt.Errorf("upgrade: preserve options failed: %w", err)) + return + } + } if hasSelfPID { if err := runtime.SetFramePID(upgradeCtx, selfPID); err != nil { proc.Process.Close() @@ -543,6 +588,9 @@ func (w *Worker) executeOne(proc *Processor) { w.scheduler.complete(proc, nil, fmt.Errorf("upgrade: init failed: %w", err)) return } + if counter, ok := newProc.(process.StepAccounted); ok { + counter.ResumeStepCount(stepsUsed) + } proc.ctx = upgradeCtx // Re-publish the out-of-band snapshot so future invalidations classify // the process by its (possibly new) upgraded source. The queue @@ -585,7 +633,7 @@ func (w *Worker) dispatchYields(ctx context.Context, proc *Processor, yields []p if preempted { if proc.casState(StateRunning, StateReady) { - w.scheduler.requeuePreempted(proc) + w.requeuePreempted(proc) } return } From 36e2282a196877e00cd7c18b9dd9a968256cbc4c Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Fri, 2 Oct 2026 20:13:03 -0400 Subject: [PATCH 03/50] fix(lua): bound module top-level code and exhausted-step work under the tick budget Module chunks run inside the main task of the first step instead of Init, so they are scheduled and preempted like any other Lua code while the exported method cache keeps serving later executions. An exhausted budget ends the step without walking the run queue, and the preempted coroutine queues behind the waiting ones. A step rejected by max_steps releases the retention leases of the messages it was delivered. --- runtime/lua/engine/preempt_test.go | 284 ++++++++++++++++++++++++ runtime/lua/engine/process.go | 222 +++++++++--------- runtime/lua/engine/process_test.go | 59 ++++- runtime/lua/engine/raised_error_test.go | 22 +- 4 files changed, 457 insertions(+), 130 deletions(-) diff --git a/runtime/lua/engine/preempt_test.go b/runtime/lua/engine/preempt_test.go index 008a160dc..ed13134d3 100644 --- a/runtime/lua/engine/preempt_test.go +++ b/runtime/lua/engine/preempt_test.go @@ -5,6 +5,8 @@ package engine import ( "context" "errors" + "fmt" + "sync/atomic" "testing" "time" @@ -12,6 +14,8 @@ import ( "github.com/wippyai/runtime/api/attrs" ctxapi "github.com/wippyai/runtime/api/context" "github.com/wippyai/runtime/api/process" + "github.com/wippyai/runtime/api/registry" + "github.com/wippyai/runtime/api/relay" "github.com/wippyai/runtime/api/runtime" luaapi "github.com/wippyai/runtime/api/runtime/lua" scheduler "github.com/wippyai/runtime/system/scheduler/actor" @@ -345,3 +349,283 @@ func TestProcessMaxStepsFailsProcess(t *testing.T) { t.Fatalf("expected ErrStepLimitExceeded, got %v", err) } } + +type countingLease struct{ released atomic.Int32 } + +func (l *countingLease) Release() { l.released.Add(1) } + +// A step that exceeds max_steps fails the process, and the retention leases of +// the messages delivered with that step are released. +func TestProcessMaxStepsReleasesDeliveredMessages(t *testing.T) { + script := ` + local s = 0 + for i = 1, 100000 do s = s + i end + return s + ` + entry := luaapi.ExecutionBudgets{TickBudget: 100, TickBudgetSet: true, MaxSteps: 1, MaxStepsSet: true} + proc, err := initWithSpawnOptions(t, script, entry, nil) + if err != nil { + t.Fatal(err) + } + var output process.StepOutput + if err := proc.Step(nil, &output); err != nil { + t.Fatal(err) + } + + lease := &countingLease{} + msg := relay.AcquireMessage() + msg.Topic = "inbox" + msg.SetRetentionLease(lease) + pkg := relay.AcquirePackage() + pkg.Messages = append(pkg.Messages, msg) + + output.Reset() + err = proc.Step([]process.Event{{Type: process.EventMessage, Data: pkg}}, &output) + if !errors.Is(err, process.ErrStepLimitExceeded) { + t.Fatalf("expected ErrStepLimitExceeded, got %v", err) + } + proc.Close() + if got := lease.released.Load(); got != 1 { + t.Fatalf("expected the retention lease to be released once, got %d", got) + } +} + +// Tasks that did not get a turn in an exhausted step stay queued in order; the +// step does not walk the run queue. +func TestProcessExhaustedBudgetKeepsWaitingTasksQueued(t *testing.T) { + proc := initPreemptProcess(t, ` + local release = false + for _ = 1, 50 do + coroutine.spawn(function() while not release do end end) + end + while true do coroutine.yield() end + `, 20) + + var output process.StepOutput + if err := proc.Step(nil, &output); err != nil { + t.Fatal(err) + } + if output.Status() != process.StepPreempted { + t.Fatalf("expected StepPreempted, got %v", output.Status()) + } + if proc.preempted != nil { + t.Fatal("the preempted task must be back in the run queue at the end of the step") + } + if got := proc.queue.Len(); got != len(proc.threads) { + t.Fatalf("expected every live task queued (%d threads), got %d queued", len(proc.threads), got) + } +} + +// Module top-level code runs inside the first step under the tick budget, so +// Init returns at once and an endless chunk is preempted like any other loop. +func TestProcessModuleTopLevelIsPreempted(t *testing.T) { + proc := mustNewProcess(t, WithScript(` + while true do end + return { main = function() end } + `, "module.lua"), WithProcessExecutionBudgets(luaapi.ExecutionBudgets{TickBudget: 100, TickBudgetSet: true})) + proc.EnablePreemption() + t.Cleanup(proc.Close) + ctx, _ := ctxapi.OpenFrameContext(context.Background()) + + inited := make(chan error, 1) + go func() { inited <- proc.Init(ctx, "main", nil) }() + select { + case err := <-inited: + if err != nil { + t.Fatal(err) + } + case <-time.After(5 * time.Second): + t.Fatal("Init blocked on module top-level code") + } + + var output process.StepOutput + if err := proc.Step(nil, &output); err != nil { + t.Fatal(err) + } + if output.Status() != process.StepPreempted { + t.Fatalf("expected the module chunk to be preempted, got %v", output.Status()) + } +} + +// A module whose top level never ends does not block the goroutine that +// submits it to the scheduler. +func TestSubmitDoesNotRunModuleTopLevelOnCaller(t *testing.T) { + executor := newBenchExecutorWithOptions(newMockRegistry(), scheduler.WithWorkers(1)) + executor.Start() + defer executor.Stop() + + proc := mustNewProcess(t, WithScript(` + while true do end + return { main = function() end } + `, "module.lua")) + ctx, cancelProc := context.WithCancel(context.Background()) + defer cancelProc() + frame, _ := ctxapi.OpenFrameContext(ctx) + + submitted := make(chan error, 1) + go func() { + _, err := executor.sched.Submit(frame, newTestPID("module-spin"), &mockProcess{Process: proc}, "main", nil) + submitted <- err + }() + select { + case err := <-submitted: + if err != nil { + t.Fatal(err) + } + case <-time.After(5 * time.Second): + t.Fatal("Submit blocked on module top-level code") + } +} + +// The module chunk runs once per process; later executions of the same +// process reuse the cached method. +func TestProcessReusesCachedModuleAcrossExecutions(t *testing.T) { + proc := mustNewProcess(t, WithScript(` + runs = (runs or 0) + 1 + return { main = function() return runs end } + `, "module.lua")) + t.Cleanup(proc.Close) + + for i := 0; i < 3; i++ { + ctx, _ := ctxapi.OpenFrameContext(context.Background()) + if err := proc.Init(ctx, "main", nil); err != nil { + t.Fatalf("execution %d: %v", i, err) + } + var output process.StepOutput + if err := proc.Step(nil, &output); err != nil { + t.Fatalf("execution %d: %v", i, err) + } + if output.Status() != process.StepDone { + t.Fatalf("execution %d: status %v", i, output.Status()) + } + if v, _ := output.Result().Data().(lua.LValue); lua.LVAsNumber(v) != 1 { + t.Fatalf("execution %d: expected the module chunk to have run once, got %v", i, output.Result().Data()) + } + } +} + +func TestProcessStepCountContinuesAcrossResume(t *testing.T) { + script := ` + local s = 0 + for i = 1, 100000 do s = s + i end + return s + ` + entry := luaapi.ExecutionBudgets{TickBudget: 100, TickBudgetSet: true, MaxSteps: 5, MaxStepsSet: true} + proc, err := initWithSpawnOptions(t, script, entry, nil) + if err != nil { + t.Fatal(err) + } + proc.ResumeStepCount(4) + if proc.StepsUsed() != 4 { + t.Fatalf("expected 4 steps used, got %d", proc.StepsUsed()) + } + var output process.StepOutput + if err := proc.Step(nil, &output); err != nil { + t.Fatalf("the fifth step is within the limit: %v", err) + } + output.Reset() + if err := proc.Step(nil, &output); !errors.Is(err, process.ErrStepLimitExceeded) { + t.Fatalf("expected ErrStepLimitExceeded, got %v", err) + } +} + +type upgradeFactory struct { + create func() (process.Process, *process.Meta, error) +} + +func (f upgradeFactory) Create(registry.ID) (process.Process, *process.Meta, error) { + return f.create() +} + +// An actor spawned with max_steps keeps that cap when it upgrades into a definition that never finishes. +func TestUpgradeKeepsSpawnMaxSteps(t *testing.T) { + executor := newBenchExecutorWithOptions(newMockRegistry(), scheduler.WithWorkers(1)) + executor.Start() + defer executor.Stop() + + upgradeReq := &UpgradeRequest{Source: registry.NewID("test", "next")} + source := mustNewProcess(t, + WithScript(` + local s = 0 + for i = 1, 300 do s = s + i end + coroutine.yield(upgrade_request) + `, "source.lua"), + WithProcessExecutionBudgets(luaapi.ExecutionBudgets{TickBudget: 100, TickBudgetSet: true}), + WithModuleBinder(wrapBinder(func(l *lua.LState) { l.SetGlobal("upgrade_request", upgradeReq) })), + ) + + appCtx := ctxapi.WithAppContext(context.Background(), ctxapi.NewAppContext()) + process.WithFactory(appCtx, upgradeFactory{create: func() (process.Process, *process.Meta, error) { + next, err := NewProcess(WithScript(`while true do end`, "next.lua"), + WithProcessExecutionBudgets(luaapi.ExecutionBudgets{TickBudget: 100, TickBudgetSet: true})) + if err != nil { + return nil, nil, err + } + return &mockProcess{Process: next}, &process.Meta{}, nil + }}) + + ctx, cancel := context.WithTimeout(appCtx, 10*time.Second) + defer cancel() + frame, fc := ctxapi.OpenFrameContext(ctx) + if err := runtime.SetFrameID(frame, registry.NewID("test", "source")); err != nil { + t.Fatal(err) + } + if err := fc.Set(runtime.FrameLifecycleOptionsKey, attrs.Attributes(attrs.Bag{"max_steps": int64(20)})); err != nil { + t.Fatal(err) + } + + result, err := executor.Execute(frame, newTestPID("upgrader"), &mockProcess{Process: source}, "", nil) + if err != nil { + t.Fatal(err) + } + if !errors.Is(result.Error, process.ErrStepLimitExceeded) { + t.Fatalf("expected the upgraded actor to fail with ErrStepLimitExceeded, got %v", result.Error) + } +} + +// A coroutine that keeps hitting its budget does not starve the actor's other +// coroutines: preempted tasks queue behind the ones still waiting. +func TestProcessPreemptedCoroutineDoesNotStarveOthers(t *testing.T) { + proc := initPreemptProcess(t, ` + coroutine.spawn(function() while true do end end) + local s = 0 + for i = 1, 20000 do s = s + i end + return s + `, 200) + + v, preempts := stepPreempted(t, proc, 5000) + if lua.LVAsNumber(v) != 200010000 { + t.Fatalf("expected 200010000, got %v", v) + } + if preempts == 0 { + t.Fatal("expected preemption") + } +} + +func BenchmarkPreemptedStepManyCoroutines(b *testing.B) { + for _, n := range []int{10, 100, 1000} { + b.Run(fmt.Sprintf("tasks=%d", n), func(b *testing.B) { + proc := mustNewProcess(b, WithScript(fmt.Sprintf(` + for _ = 1, %d do + coroutine.spawn(function() while true do for _ = 1, 1000 do end coroutine.yield() end end) + end + while true do coroutine.yield() end + `, n), "many.lua"), WithProcessExecutionBudgets(luaapi.ExecutionBudgets{TickBudget: 64, TickBudgetSet: true})) + proc.EnablePreemption() + ctx, _ := ctxapi.OpenFrameContext(context.Background()) + if err := proc.Init(ctx, "", nil); err != nil { + b.Fatal(err) + } + defer proc.Close() + var output process.StepOutput + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + output.Reset() + if err := proc.Step(nil, &output); err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/runtime/lua/engine/process.go b/runtime/lua/engine/process.go index acb2bfe57..d9c9f3c13 100644 --- a/runtime/lua/engine/process.go +++ b/runtime/lua/engine/process.go @@ -101,10 +101,13 @@ type Process struct { messageQueueLimits map[string]int64 messageQueueItemLimits map[string]int mainTask *Task - upgradeRequest *UpgradeRequest - proto *lua.FunctionProto - queue *TaskQueue - factory *Factory + // preempted is the task the VM suspended when the step's tick budget ran + // out; it re-enters the run queue behind the tasks still waiting. + preempted *Task + upgradeRequest *UpgradeRequest + proto *lua.FunctionProto + queue *TaskQueue + factory *Factory // pendingOutdated holds the single coalesced OUTDATED event awaiting // delivery to an upgradable process's events channel. Nil when none pending. pendingOutdated *topology.OutdatedEvent @@ -115,8 +118,11 @@ type Process struct { // stalledChans records channels that could not accept a message during the // current flush. Later messages for the same channel, including terminals, // remain queued so delivery order cannot be inverted. - stalledChans map[*Channel]struct{} - exported map[string]*lua.LFunction + stalledChans map[*Channel]struct{} + exported map[string]*lua.LFunction + // exportFn is the Go function the module bootstrap calls to cache the + // requested method; created on first use. + exportFn *lua.LFunction messageQueueDiscarded map[string]struct{} messageQueueOverflowed map[string]struct{} channelQueue *TaskQueue @@ -132,10 +138,7 @@ type Process struct { externalTasks []*Task outTasks []*Task threads []*Task - // deferred holds runnable tasks that did not finish within this step's - // preemption budget; they run again in the next step. - deferred []*Task - yieldSeq uint64 + yieldSeq uint64 // entryBudgets are the execution options of the process entry; spawn // options override them at Init. entryBudgets luaapi.ExecutionBudgets @@ -619,35 +622,24 @@ func (p *Process) Init(ctx context.Context, method string, input payload.Payload // Determine which function to execute var fn *lua.LFunction + var bootstrap []lua.LValue - // If method specified, try to use cached function or extract from module if method != "" { - if p.exported != nil { - if cached, ok := p.exported[method]; ok { - fn = cached - } - } - - // No cached function - need to run script to get module and extract method + fn = p.exported[method] if fn == nil { - if err := p.extractMethod(method); err != nil { + // The module chunk runs inside the main task so its execution is + // scheduled and budgeted like any other Lua code. + module, err := p.loadMain() + if err != nil { return err } - fn = p.exported[method] + fn = p.state.LoadProto(moduleBootstrapProto) + bootstrap = []lua.LValue{module, p.exportFunction(), lua.LString(method)} } } else { - // No method - run the script directly (legacy behavior) - - if p.proto != nil { - fn = p.state.LoadProto(p.proto) - } else if p.script != "" { - var err error - fn, err = p.state.Load(strings.NewReader(p.script), p.scriptName) - if err != nil { - return runtimelua.NewLoadScriptError(err) - } - } else { - return luaapi.ErrNoScriptOrProto + var err error + if fn, err = p.loadMain(); err != nil { + return err } } @@ -655,8 +647,9 @@ func (p *Process) Init(ctx context.Context, method string, input payload.Payload p.mainTask = p.createTask(fn) // Convert input payloads to Lua values as arguments - if len(input) > 0 { - args := make([]lua.LValue, 0, len(input)) + if len(input) > 0 || bootstrap != nil { + args := make([]lua.LValue, 0, len(bootstrap)+len(input)) + args = append(args, bootstrap...) for _, pl := range input { args = append(args, transcodeToLua(ctx, pl)) } @@ -666,78 +659,76 @@ func (p *Process) Init(ctx context.Context, method string, input payload.Payload return nil } -// extractMethod runs the script to get module table and extracts the method function. -func (p *Process) extractMethod(method string) error { - // Load script function - var scriptFn *lua.LFunction +// moduleBootstrapProto runs a module chunk and calls the exported method with +// the process input. It receives the chunk function, the export function and +// the method name ahead of the input arguments. +var moduleBootstrapProto = mustCompileBootstrap(` +local chunk, export, method = ... +return export(method, (chunk()))(select(4, ...)) +`, "=module_bootstrap") - if p.proto != nil { - scriptFn = p.state.LoadProto(p.proto) - } else if p.script != "" { - var err error - scriptFn, err = p.state.Load(strings.NewReader(p.script), p.scriptName) - if err != nil { - return runtimelua.NewLoadScriptError(err) - } - } else { - return luaapi.ErrNoScriptOrProto +func mustCompileBootstrap(source, name string) *lua.FunctionProto { + proto, err := lua.CompileString(source, name) + if err != nil { + panic(err) } + return proto +} - // Run script synchronously to get module table - if err := p.state.CallByParam(lua.P{ - Fn: scriptFn, - NRet: 1, - Protect: true, - }); err != nil { - var raised *lua.Error - if errors.As(err, &raised) { - return toAPIError(raised) +// loadMain loads the process script as a function without running it. +func (p *Process) loadMain() (*lua.LFunction, error) { + switch { + case p.proto != nil: + return p.state.LoadProto(p.proto), nil + case p.script != "": + fn, err := p.state.Load(strings.NewReader(p.script), p.scriptName) + if err != nil { + return nil, runtimelua.NewLoadScriptError(err) } - return runtimelua.NewExecuteScriptError(err) + return fn, nil + default: + return nil, luaapi.ErrNoScriptOrProto } +} - // Get return value - ret := p.state.Get(-1) - p.state.Pop(1) - - // Initialize exported map - if p.exported == nil { - p.exported = make(map[string]*lua.LFunction) +// exportFunction returns the Go function that resolves a method from the value +// a module chunk returned and caches it for later executions of this process. +func (p *Process) exportFunction() *lua.LFunction { + if p.exportFn == nil { + p.exportFn = p.state.NewFunction(p.exportMethod) } + return p.exportFn +} + +// exportMethod takes (method, moduleValue) and returns the method function. +// The module value is either the function itself or a table holding it. +func (p *Process) exportMethod(l *lua.LState) int { + method := l.CheckString(1) - // Extract function from return value var fn *lua.LFunction - switch v := ret.(type) { + switch v := l.Get(2).(type) { case *lua.LFunction: - // Script returned function directly fn = v case *lua.LTable: - // Script returned module table - extract method by name - val := v.RawGetString(method) - if val.Type() == lua.LTFunction { - fn = val.(*lua.LFunction) - } + fn, _ = v.RawGetString(method).(*lua.LFunction) } - if fn == nil { - return runtimelua.NewMethodNotFoundError(method) + l.Error(lua.WrapError(runtimelua.NewMethodNotFoundError(method), "").WithKind(lua.NotFound).WithRetryable(false), 1) + return 0 } + if p.exported == nil { + p.exported = make(map[string]*lua.LFunction) + } p.exported[method] = fn - return nil + l.Push(fn) + return 1 } // Step advances the process by one iteration. // events contains yield completions and messages from the scheduler. // out is the scheduler-owned buffer where the process writes yields and status. func (p *Process) Step(events []process.Event, out *process.StepOutput) error { - if err := p.chargeStep(); err != nil { - p.clearExecution() - out.Done(nil) - return err - } - p.state.SetTickBudget(p.stepTickBudget()) - // Collect messages from events var messages []*relay.Package for _, ev := range events { @@ -772,6 +763,16 @@ func (p *Process) Step(events []process.Event, out *process.StepOutput) error { relay.ReleasePackage(pkg) } + // Events are consumed before the step is charged, so a step that exceeds + // its limit leaves their retention leases with the mailbox, which releases + // them when the process closes. + if err := p.chargeStep(); err != nil { + p.clearExecution() + out.Done(nil) + return err + } + p.state.SetTickBudget(p.stepTickBudget()) + // Process in a loop until stable (no new subscriptions trigger channel work) var externalTasks []*Task var err error @@ -816,6 +817,12 @@ func (p *Process) Step(events []process.Event, out *process.StepOutput) error { // with only the unhandled tasks (subscription yields were removed) p.externalTasks = externalTasks + // An exhausted tick budget ends the step; tasks still queued run in + // the next one. + if p.state.TickBudget() == 0 { + break + } + // Continue looping if subscriptions were handled (may have added tasks) // or if queue has tasks to process. // @@ -835,13 +842,13 @@ func (p *Process) Step(events []process.Event, out *process.StepOutput) error { } } - // Tasks that ran out of the step's budget run again next step. - preempted := len(p.deferred) > 0 - for _, task := range p.deferred { - p.queue.Push(task) + // The VM-preempted task runs again behind the tasks that did not get a + // turn in this step. + if p.preempted != nil { + p.queue.Push(p.preempted) + p.preempted = nil } - clear(p.deferred) - p.deferred = p.deferred[:0] + preempted := p.state.TickBudget() == 0 && !p.queue.IsEmpty() // Check for upgrade request if p.upgradeRequest != nil { @@ -953,6 +960,17 @@ func (p *Process) chargeStep() error { return nil } +// StepsUsed implements process.StepAccounted. +func (p *Process) StepsUsed() uint64 { + return p.steps +} + +// ResumeStepCount implements process.StepAccounted. The count continues from +// used, so max_steps bounds the actor across upgrades. +func (p *Process) ResumeStepCount(used uint64) { + p.steps = used +} + // State returns the underlying Lua state. func (p *Process) State() *lua.LState { return p.state @@ -988,12 +1006,8 @@ func (p *Process) processChannelYields() ([]*Task, error) { channels := p.channels channelQueue := p.ChannelQueue() - // Transfer tasks from process queue to channel queue - for _, task := range p.queue.Drain() { - channelQueue.Push(task) - } - - // Process all queued tasks + // Tasks already in the process queue run first; tasks that channel + // operations wake follow through the channel queue. boot := true for !channelQueue.IsEmpty() || boot { boot = false @@ -1785,7 +1799,7 @@ func (p *Process) vmStep(tasks ...*Task) ([]*Task, error) { // Reuse yield buffer p.yieldBuf = p.yieldBuf[:0] - for !p.queue.IsEmpty() { + for !p.queue.IsEmpty() && p.state.TickBudget() != 0 { task := p.queue.Pop() if task == nil { continue @@ -1794,10 +1808,6 @@ func (p *Process) vmStep(tasks ...*Task) ([]*Task, error) { if task.State != lua.ResumeYield && task.State != lua.ResumePreempted { continue } - if p.state.TickBudget() == 0 { - p.deferred = append(p.deferred, task) - continue - } thread := task.Thread() state, values, err := p.state.ResumeInto(thread, task.Function(), task.retBuf, task.Resumed...) @@ -1847,7 +1857,7 @@ func (p *Process) vmStep(tasks ...*Task) ([]*Task, error) { } p.removeTask(task) case lua.ResumePreempted: - p.deferred = append(p.deferred, task) + p.preempted = task case lua.ResumeError: p.removeTask(task) } @@ -1893,8 +1903,7 @@ func (p *Process) killAllThreads() { } p.threads = p.threads[:0] p.queue.Drain() - clear(p.deferred) - p.deferred = p.deferred[:0] + p.preempted = nil } // handleSpawnRequest checks if yielded values contain a SpawnRequest and handles it. @@ -1984,8 +1993,7 @@ func (p *Process) Close() { p.yieldBuf = p.yieldBuf[:0] p.externalTasks = p.externalTasks[:0] p.outTasks = p.outTasks[:0] - clear(p.deferred) - p.deferred = p.deferred[:0] + p.preempted = nil p.clearMessageQueue() // Clear all references @@ -1999,6 +2007,7 @@ func (p *Process) Close() { p.factory = nil p.pendingYields = nil p.exported = nil + p.exportFn = nil p.channelQueue = nil p.channels = nil p.subs = nil @@ -2130,8 +2139,7 @@ func (p *Process) clearExecution() { // Drain queue p.queue.Drain() - clear(p.deferred) - p.deferred = p.deferred[:0] + p.preempted = nil // Clear main task reference and result p.mainTask = nil diff --git a/runtime/lua/engine/process_test.go b/runtime/lua/engine/process_test.go index 792d7fbe7..6bcdbda3e 100644 --- a/runtime/lua/engine/process_test.go +++ b/runtime/lua/engine/process_test.go @@ -1740,6 +1740,21 @@ func TestProcessInitNoScript(t *testing.T) { } } +// requireMethodNotFound checks that err carries the method-not-found error +// in its chain. +func requireMethodNotFound(t *testing.T, err error) { + t.Helper() + if err == nil { + t.Fatal("expected a method-not-found error") + } + for _, link := range apierror.BuildChain(err).Errors { + if link.Kind == string(apierror.NotFound) { + return + } + } + t.Fatalf("expected a not-found error in the chain, got %v", err) +} + // TestProcessInitInvalidMethod tests Init with non-existent method func TestProcessInitInvalidMethod(t *testing.T) { script := `return {}` @@ -1747,12 +1762,14 @@ func TestProcessInitInvalidMethod(t *testing.T) { proc := mustNewProcess(t, WithScript(script, "test.lua")) ctx, _ := ctxapi.OpenFrameContext(context.Background()) - err := proc.Init(ctx, "nonexistent", nil) - - if err == nil { - t.Fatal("expected error for missing method") + if err := proc.Init(ctx, "nonexistent", nil); err != nil { + t.Fatal(err) } - proc.Close() + defer proc.Close() + + var output process.StepOutput + err := proc.Step(nil, &output) + requireMethodNotFound(t, err) } // TestProcessInitSyntaxError tests Init with malformed Lua @@ -3828,6 +3845,11 @@ func TestProcessExtractMethodWithScript(t *testing.T) { } defer proc.Close() + var output process.StepOutput + if err := proc.Step(nil, &output); err != nil { + t.Fatal(err) + } + // Verify method was extracted if proc.exported == nil || proc.exported["handle"] == nil { t.Fatal("handle method should be extracted") @@ -3840,9 +3862,16 @@ func TestProcessExtractMethodNotFound(t *testing.T) { proc := mustNewProcess(t, WithScript(script, "test.lua")) ctx, _ := ctxapi.OpenFrameContext(context.Background()) - err := proc.Init(ctx, "nonexistent", nil) - if err == nil { - t.Fatal("expected error for missing method") + if err := proc.Init(ctx, "nonexistent", nil); err != nil { + t.Fatal(err) + } + defer proc.Close() + + var output process.StepOutput + err := proc.Step(nil, &output) + requireMethodNotFound(t, err) + if output.Status() != process.StepDone { + t.Fatalf("expected the failed process to be done, got %v", output.Status()) } } @@ -3857,6 +3886,11 @@ func TestProcessExtractMethodScriptReturnsFunction(t *testing.T) { } defer proc.Close() + var output process.StepOutput + if err := proc.Step(nil, &output); err != nil { + t.Fatal(err) + } + // Should work - direct function return gets stored if proc.exported == nil || proc.exported["handle"] == nil { t.Fatal("direct function should be extracted") @@ -3869,8 +3903,13 @@ func TestProcessExtractMethodScriptError(t *testing.T) { proc := mustNewProcess(t, WithScript(script, "test.lua")) ctx, _ := ctxapi.OpenFrameContext(context.Background()) - err := proc.Init(ctx, "handle", nil) - if err == nil { + if err := proc.Init(ctx, "handle", nil); err != nil { + t.Fatal(err) + } + defer proc.Close() + + var output process.StepOutput + if err := proc.Step(nil, &output); err == nil { t.Fatal("expected error from script execution") } } diff --git a/runtime/lua/engine/raised_error_test.go b/runtime/lua/engine/raised_error_test.go index 5903d3d1d..707f2fb3e 100644 --- a/runtime/lua/engine/raised_error_test.go +++ b/runtime/lua/engine/raised_error_test.go @@ -17,10 +17,9 @@ func TestProcessRaisedTypedError(t *testing.T) { for _, tc := range []struct { name string script string - init bool }{ - {"body", `return {main=function() error(errors.new({message="bad declaration", kind=errors.INVALID, retryable=false, details={field="target"}})) end}`, false}, - {"initialization", `error(errors.new({message="bad declaration", kind=errors.INVALID, retryable=false, details={field="target"}}))`, true}, + {"body", `return {main=function() error(errors.new({message="bad declaration", kind=errors.INVALID, retryable=false, details={field="target"}})) end}`}, + {"initialization", `error(errors.new({message="bad declaration", kind=errors.INVALID, retryable=false, details={field="target"}}))`}, } { t.Run(tc.name, func(t *testing.T) { factory := NewFactory(FactoryConfig{Script: tc.script, ScriptName: "raised.lua", ModuleBinders: []ModuleBinder{wrapBinder(func(l *lua.LState) { lua.OpenErrors(l) })}}) @@ -31,16 +30,13 @@ func TestProcessRaisedTypedError(t *testing.T) { proc := p.(*Process) defer proc.Close() ctx, _ := ctxapi.OpenFrameContext(context.Background()) - err = proc.Init(ctx, "main", nil) - if !tc.init { - if err != nil { - t.Fatal(err) - } - var output process.StepOutput - err = proc.Step(nil, &output) - if output.Status() != process.StepDone { - t.Fatalf("status = %v", output.Status()) - } + if err := proc.Init(ctx, "main", nil); err != nil { + t.Fatal(err) + } + var output process.StepOutput + err = proc.Step(nil, &output) + if output.Status() != process.StepDone { + t.Fatalf("status = %v", output.Status()) } var apiErr apierror.Error if !errors.As(err, &apiErr) { From d83d14f4ffaef1e1e9d5ca85a79d9f98454ee695 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Fri, 2 Oct 2026 20:39:38 -0400 Subject: [PATCH 04/50] test(lua): close the step-limit lease test process once The test closed its process explicitly and again through the helper's cleanup, putting one Process into the shared pool twice; a later test could then receive a Process whose state another test released. --- runtime/lua/engine/preempt_test.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/runtime/lua/engine/preempt_test.go b/runtime/lua/engine/preempt_test.go index ed13134d3..a9c58f18b 100644 --- a/runtime/lua/engine/preempt_test.go +++ b/runtime/lua/engine/preempt_test.go @@ -363,8 +363,12 @@ func TestProcessMaxStepsReleasesDeliveredMessages(t *testing.T) { return s ` entry := luaapi.ExecutionBudgets{TickBudget: 100, TickBudgetSet: true, MaxSteps: 1, MaxStepsSet: true} - proc, err := initWithSpawnOptions(t, script, entry, nil) - if err != nil { + // The test closes the process itself to observe the lease release, so it + // does not use initWithSpawnOptions, which closes it at cleanup. + proc := mustNewProcess(t, WithScript(script, "preempt.lua"), WithProcessExecutionBudgets(entry)) + proc.EnablePreemption() + ctx, _ := ctxapi.OpenFrameContext(context.Background()) + if err := proc.Init(ctx, "", nil); err != nil { t.Fatal(err) } var output process.StepOutput @@ -380,7 +384,7 @@ func TestProcessMaxStepsReleasesDeliveredMessages(t *testing.T) { pkg.Messages = append(pkg.Messages, msg) output.Reset() - err = proc.Step([]process.Event{{Type: process.EventMessage, Data: pkg}}, &output) + err := proc.Step([]process.Event{{Type: process.EventMessage, Data: pkg}}, &output) if !errors.Is(err, process.ErrStepLimitExceeded) { t.Fatalf("expected ErrStepLimitExceeded, got %v", err) } From 385ceef5d88009db3461c2c722181a27182ed61f Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Fri, 2 Oct 2026 20:56:19 -0400 Subject: [PATCH 05/50] fix(lua): deliver queued messages when a step exhausts its tick budget A receive that parks on the step's last tick now sees the messages held back for it, so the process reports preempted instead of idle. --- runtime/lua/engine/budget_flush_test.go | 68 +++++++++++++++++++++++++ runtime/lua/engine/process.go | 9 +++- 2 files changed, 75 insertions(+), 2 deletions(-) create mode 100644 runtime/lua/engine/budget_flush_test.go diff --git a/runtime/lua/engine/budget_flush_test.go b/runtime/lua/engine/budget_flush_test.go new file mode 100644 index 000000000..e4233d217 --- /dev/null +++ b/runtime/lua/engine/budget_flush_test.go @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: MPL-2.0 + +package engine + +import ( + "fmt" + "testing" + + lua "github.com/wippyai/go-lua" + ctxapi "github.com/wippyai/runtime/api/context" + "github.com/wippyai/runtime/api/payload" + "github.com/wippyai/runtime/api/process" + "github.com/wippyai/runtime/api/relay" + luaapi "github.com/wippyai/runtime/api/runtime/lua" +) + +// A message held back by a full channel is delivered to the receiver that +// parks on it even when that receive uses up the step's last tick: the +// process never reports idle while a queued message matches a parked receiver. +func TestProcessExhaustedBudgetDeliversQueuedMessage(t *testing.T) { + for budget := int64(1); budget <= 40; budget++ { + t.Run(fmt.Sprintf("budget=%d", budget), func(t *testing.T) { + proto, err := lua.CompileString(` + local ch = channel.new(1) + subscribe("inbox", ch) + local a = ch:receive() + local b = ch:receive() + return b + `, "receive.lua") + if err != nil { + t.Fatal(err) + } + proc := mustNewProcess(t, WithProto(proto), WithProcessExecutionBudgets(luaapi.ExecutionBudgets{TickBudget: budget, TickBudgetSet: true})) + proc.EnablePreemption() + t.Cleanup(proc.Close) + ctx, _ := ctxapi.OpenFrameContext(ctxapi.NewRootContext()) + ctx = payload.WithTranscoder(ctx, createInboxTestTranscoder()) + if err := proc.Init(ctx, "", nil); err != nil { + t.Fatal(err) + } + LoadModuleDef(proc.State(), ChannelModule) + loadPubSubGlobals(proc.State()) + + var output process.StepOutput + events := []process.Event{{ + Type: process.EventMessage, + Data: &relay.Package{Messages: []*relay.Message{ + {Topic: "inbox", Payloads: payload.Payloads{payload.NewPayload(lua.LString("first"), payload.Lua)}}, + {Topic: "inbox", Payloads: payload.Payloads{payload.NewPayload(lua.LString("second"), payload.Lua)}}, + }}, + }} + for i := 0; i < 200; i++ { + output.Reset() + if err := proc.Step(events, &output); err != nil { + t.Fatal(err) + } + events = nil + switch output.Status() { + case process.StepDone: + return + case process.StepIdle: + t.Fatalf("step %d: idle with a queued message for the parked receiver", i) + } + } + t.Fatal("process did not complete") + }) + } +} diff --git a/runtime/lua/engine/process.go b/runtime/lua/engine/process.go index d9c9f3c13..0f004c875 100644 --- a/runtime/lua/engine/process.go +++ b/runtime/lua/engine/process.go @@ -818,8 +818,13 @@ func (p *Process) Step(events []process.Event, out *process.StepOutput) error { p.externalTasks = externalTasks // An exhausted tick budget ends the step; tasks still queued run in - // the next one. + // the next one. Messages held back for a receiver that parked in this + // step are delivered first, so the step does not report idle while a + // queued message matches a parked receiver. if p.state.TickBudget() == 0 { + if p.subs != nil { + p.flushMessageQueue(p.subs) + } break } @@ -2300,4 +2305,4 @@ func toAPIError(err error) error { return builder.WithCause(toAPIError(inner)) } return builder -} +} \ No newline at end of file From 2f497b4c8021aac7cd6cf3b3a031e277f4090c1a Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Fri, 2 Oct 2026 21:06:35 -0400 Subject: [PATCH 06/50] fix(lua): run dependency chunks inside the budgeted main task Library chunks no longer execute through PCall while the factory builds the state. A binder registers them with DeferInitializer; the entry bootstrap runs them as Lua frames of the main task in dependency order, so loops in library code are preempted and cannot block the spawner or an upgrade. Chunk environments resolve imports at lookup time. The bootstrap uses no guest-visible global. --- runtime/lua/engine/bootstrap_test.go | 37 ++++++ runtime/lua/engine/factory_test.go | 16 ++- runtime/lua/engine/initializer.go | 86 ++++++++++++ runtime/lua/engine/process.go | 61 ++++++--- runtime/lua/engine/process_factory.go | 34 +++-- runtime/lua/engine/process_factory_test.go | 132 ++++++++++++++++++- runtime/lua/engine/process_isolation_test.go | 16 +++ 7 files changed, 342 insertions(+), 40 deletions(-) create mode 100644 runtime/lua/engine/bootstrap_test.go create mode 100644 runtime/lua/engine/initializer.go diff --git a/runtime/lua/engine/bootstrap_test.go b/runtime/lua/engine/bootstrap_test.go new file mode 100644 index 000000000..53f6fb02f --- /dev/null +++ b/runtime/lua/engine/bootstrap_test.go @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: MPL-2.0 + +package engine + +import ( + "context" + "testing" + + lua "github.com/wippyai/go-lua" + ctxapi "github.com/wippyai/runtime/api/context" + "github.com/wippyai/runtime/api/payload" + "github.com/wippyai/runtime/api/process" +) + +// The module bootstrap does not use globals the guest can change. +func TestProcessModuleBootstrapIgnoresShadowedGlobals(t *testing.T) { + proc := mustNewProcess(t, WithScript(` + select = nil + pcall = nil + unpack = nil + table = nil + return { main = function(a, b) return a + b end } + `, "module.lua")) + t.Cleanup(proc.Close) + ctx, _ := ctxapi.OpenFrameContext(context.Background()) + args := payload.Payloads{&testPayload{val: lua.LNumber(10)}, &testPayload{val: lua.LNumber(20)}} + if err := proc.Init(ctx, "main", args); err != nil { + t.Fatal(err) + } + var output process.StepOutput + if err := proc.Step(nil, &output); err != nil { + t.Fatal(err) + } + if v, _ := output.Result().Data().(lua.LValue); lua.LVAsNumber(v) != 30 { + t.Fatalf("expected 30, got %v", output.Result().Data()) + } +} diff --git a/runtime/lua/engine/factory_test.go b/runtime/lua/engine/factory_test.go index 53505cbf5..e99d14c57 100644 --- a/runtime/lua/engine/factory_test.go +++ b/runtime/lua/engine/factory_test.go @@ -7,6 +7,8 @@ import ( "testing" lua "github.com/wippyai/go-lua" + ctxapi "github.com/wippyai/runtime/api/context" + "github.com/wippyai/runtime/api/process" "github.com/wippyai/runtime/api/registry" luaapi "github.com/wippyai/runtime/api/runtime/lua" "github.com/wippyai/runtime/runtime/lua/code" @@ -781,8 +783,20 @@ func TestFactory_ModuleAliasing_LuaLibrary(t *testing.T) { t.Fatalf("factory() failed: %v", err) } - // Verify the alias is set correctly in the Lua state + // Library chunks run in the first step of the process. luaProc := proc.(*Process) + defer luaProc.Close() + frameCtx, _ := ctxapi.OpenFrameContext(context.Background()) + if err := luaProc.Init(frameCtx, "main", nil); err != nil { + t.Fatalf("Init failed: %v", err) + } + var output process.StepOutput + if err := luaProc.Step(nil, &output); err != nil { + t.Fatalf("Step failed: %v", err) + } + if v, _ := output.Result().Data().(lua.LValue); v == nil || v.String() != "from_utils" { + t.Fatalf("expected the entry point to read the library marker, got %v", output.Result().Data()) + } state := luaProc.State() // Imports are scoped to the chunk environment, not the shared _G. diff --git a/runtime/lua/engine/initializer.go b/runtime/lua/engine/initializer.go new file mode 100644 index 000000000..e91f27362 --- /dev/null +++ b/runtime/lua/engine/initializer.go @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: MPL-2.0 + +package engine + +import ( + lua "github.com/wippyai/go-lua" +) + +const initializersRegistryKey = "wippy.initializers" + +// Initializer is Lua code that must run before a process entry point, such as +// a library chunk whose result later chunks import. A ModuleBinder registers +// it with DeferInitializer instead of calling the function itself, so the code +// runs as a Lua frame of the process's main task: scheduled, preemptible and +// limited by the tick budget like the rest of the actor. +type Initializer struct { + // Fn is the function to run, created on the state the binder received. It + // runs without arguments. + Fn *lua.LFunction + // Done receives the first value Fn returned, or nil. It runs on the + // process's step goroutine, after Fn returns and before the next + // initializer starts. + Done func(result lua.LValue) +} + +// DeferInitializer registers init to run on l's process before its entry +// point. Initializers run in registration order, once per state: a later +// execution of the same process finds them completed. An initializer that +// fails is retried by the next execution. +// +// Call it from a ModuleBinder. Registration order must follow dependency +// order, so a library's imports are initialized before the library. +func DeferInitializer(l *lua.LState, init Initializer) { + list := initializersOf(l) + if list == nil { + list = &initializerList{} + l.G.Registry.RawSetString(initializersRegistryKey, list) + } + list.items = append(list.items, init) +} + +// initializerList is the registry-held sequence of a state's initializers. +// It implements lua.LValue so the state's registry can hold it. +type initializerList struct { + items []Initializer + // done counts completed initializers; started reports that advance has + // handed out items[done] in the current run. + done int + started bool +} + +func (*initializerList) String() string { return "" } +func (*initializerList) Type() lua.LValueType { return lua.LTUserData } + +func initializersOf(l *lua.LState) *initializerList { + list, _ := l.G.Registry.RawGetString(initializersRegistryKey).(*initializerList) + return list +} + +func (list *initializerList) pending() bool { + return list != nil && list.done < len(list.items) +} + +// begin restarts the run at the first incomplete initializer. +func (list *initializerList) begin() { + list.started = false +} + +// advance is the Go side of the bootstrap loop. It takes the previous +// initializer's result (ignored on the first call) and returns the next +// function to run, or nothing when all have run. +func (list *initializerList) advance(l *lua.LState) int { + if list.started { + item := list.items[list.done] + list.done++ + if item.Done != nil { + item.Done(l.Get(1)) + } + } + list.started = list.done < len(list.items) + if !list.started { + return 0 + } + l.Push(list.items[list.done].Fn) + return 1 +} diff --git a/runtime/lua/engine/process.go b/runtime/lua/engine/process.go index 0f004c875..8c79473f9 100644 --- a/runtime/lua/engine/process.go +++ b/runtime/lua/engine/process.go @@ -624,19 +624,31 @@ func (p *Process) Init(ctx context.Context, method string, input payload.Payload var fn *lua.LFunction var bootstrap []lua.LValue - if method != "" { + inits := initializersOf(p.state) + pending := inits.pending() + if method != "" && !pending { fn = p.exported[method] - if fn == nil { - // The module chunk runs inside the main task so its execution is - // scheduled and budgeted like any other Lua code. - module, err := p.loadMain() - if err != nil { - return err - } - fn = p.state.LoadProto(moduleBootstrapProto) - bootstrap = []lua.LValue{module, p.exportFunction(), lua.LString(method)} + } + if fn == nil && (method != "" || pending) { + // The module chunk and the initializers it depends on run inside the + // main task so their execution is scheduled and budgeted like any + // other Lua code. + chunk, err := p.loadMain() + if err != nil { + return err } - } else { + fn = p.state.LoadProto(entryBootstrapProto) + advance := p.state.NewFunction(func(*lua.LState) int { return 0 }) + if inits != nil { + inits.begin() + advance = p.state.NewFunction(inits.advance) + } + export, name := lua.LValue(lua.LFalse), lua.LValue(lua.LFalse) + if method != "" { + export, name = p.exportFunction(), lua.LString(method) + } + bootstrap = []lua.LValue{chunk, advance, export, name} + } else if fn == nil { var err error if fn, err = p.loadMain(); err != nil { return err @@ -659,13 +671,24 @@ func (p *Process) Init(ctx context.Context, method string, input payload.Payload return nil } -// moduleBootstrapProto runs a module chunk and calls the exported method with -// the process input. It receives the chunk function, the export function and -// the method name ahead of the input arguments. -var moduleBootstrapProto = mustCompileBootstrap(` -local chunk, export, method = ... -return export(method, (chunk()))(select(4, ...)) -`, "=module_bootstrap") +// entryBootstrapProto runs the deferred initializers, then the entry chunk. It +// receives the chunk function, the initializer iterator, the export function +// and the method name, followed by the process input. A false method runs the +// chunk itself with the input; otherwise the chunk returns the module the +// method is exported from. The code uses no global, so guest code cannot +// change it. +var entryBootstrapProto = mustCompileBootstrap(` +return (function(chunk, advance, export, method, ...) + local init = advance() + while init do + init = advance(init()) + end + if not method then + return chunk(...) + end + return export(method, (chunk()))(...) +end)(...) +`, "=entry_bootstrap") func mustCompileBootstrap(source, name string) *lua.FunctionProto { proto, err := lua.CompileString(source, name) @@ -2305,4 +2328,4 @@ func toAPIError(err error) error { return builder.WithCause(toAPIError(inner)) } return builder -} \ No newline at end of file +} diff --git a/runtime/lua/engine/process_factory.go b/runtime/lua/engine/process_factory.go index a57b3cf37..8b06bf5df 100644 --- a/runtime/lua/engine/process_factory.go +++ b/runtime/lua/engine/process_factory.go @@ -268,12 +268,10 @@ func (f *ProcessFactory) isolationBinder( env := buildChunkEnv(l, base, compiled.Imports[id], valueByNode) fn := l.LoadProto(dep.Proto) fn.Env = env - l.Push(fn) - if err := l.PCall(0, 1, nil); err != nil { - return fmt.Errorf("failed to load dependency %s: %w", dep.Name, toAPIError(err)) - } - valueByNode[id] = l.Get(-1) - l.Pop(1) + DeferInitializer(l, Initializer{ + Fn: fn, + Done: func(result lua.LValue) { valueByNode[id] = result }, + }) } } @@ -296,20 +294,28 @@ func (f *ProcessFactory) isolationBinder( func buildChunkEnv(l *lua.LState, base *lua.LTable, imports []code.Import, valueByNode map[registry.ID]lua.LValue) *lua.LTable { env := l.NewTable() - resolved := make(map[string]lua.LValue, len(imports)) + // Library values appear in valueByNode as their initializers complete, so + // imports resolve at lookup time. Imports sharing an alias resolve to the + // last one that has a value. + aliasIDs := make(map[string][]registry.ID, len(imports)) baseHadAlias := make(map[string]bool, len(imports)) for _, imp := range imports { - v, ok := valueByNode[imp.ID] - if !ok { - continue - } alias := imp.Alias if alias == "" { alias = imp.ID.Name } - resolved[alias] = v + aliasIDs[alias] = append(aliasIDs[alias], imp.ID) baseHadAlias[alias] = base.RawGetString(alias) != lua.LNil } + resolve := func(alias string) (lua.LValue, bool) { + ids := aliasIDs[alias] + for i := len(ids) - 1; i >= 0; i-- { + if v, ok := valueByNode[ids[i]]; ok { + return v, true + } + } + return nil, false + } mt := l.NewTable() mt.RawSetString("__index", l.NewFunction(func(s *lua.LState) int { @@ -322,7 +328,7 @@ func buildChunkEnv(l *lua.LState, base *lua.LTable, imports []code.Import, value s.Push(baseValue) return 1 } - if v, ok := resolved[name]; ok { + if v, ok := resolve(name); ok { s.Push(v) return 1 } @@ -337,7 +343,7 @@ func buildChunkEnv(l *lua.LState, base *lua.LTable, imports []code.Import, value env.RawSetString("require", l.NewFunction(func(s *lua.LState) int { name := s.CheckString(1) - if v, ok := resolved[name]; ok { + if v, ok := resolve(name); ok { s.Push(v) return 1 } diff --git a/runtime/lua/engine/process_factory_test.go b/runtime/lua/engine/process_factory_test.go index 2cdedca01..431e41a88 100644 --- a/runtime/lua/engine/process_factory_test.go +++ b/runtime/lua/engine/process_factory_test.go @@ -3,10 +3,14 @@ package engine import ( + "context" "testing" + "time" lua "github.com/wippyai/go-lua" + ctxapi "github.com/wippyai/runtime/api/context" apierror "github.com/wippyai/runtime/api/error" + "github.com/wippyai/runtime/api/process" "github.com/wippyai/runtime/api/registry" luaapi "github.com/wippyai/runtime/api/runtime/lua" "github.com/wippyai/runtime/runtime/lua/code" @@ -259,9 +263,6 @@ func TestNewProcessFactory(t *testing.T) { } func TestDependencyRaisePreservesKind(t *testing.T) { - l := lua.NewState() - defer l.Close() - lua.OpenErrors(l) id := registry.NewID("test", "dependency") compiled := &code.CompiledMain{ Dependencies: []code.CompiledProto{{ @@ -270,14 +271,133 @@ func TestDependencyRaisePreservesKind(t *testing.T) { Proto: compileTestProto(t, `error(errors.new({message="bad dependency", kind=errors.INVALID, retryable=false}))`), }}, } - pf := NewProcessFactory(nil) - binder := pf.isolationBinder(compiled, newProcessConfig(), nil, nil, nil, nil) - err := binder(l) + proc := newDependencyProcess(t, compiled, `return { main = function() end }`, 0) + if err := proc.Init(frameContext(), "main", nil); err != nil { + t.Fatal(err) + } + var output process.StepOutput + err := proc.Step(nil, &output) if err == nil || apierror.BuildChain(err).Root().Kind != string(apierror.Invalid) { t.Fatalf("dependency error = %v", err) } } +// Library chunks run in dependency order under per-chunk environments, inside +// the first step. +func TestDependenciesRunInFirstStepWithScopedEnvironments(t *testing.T) { + baseID := registry.NewID("test", "base") + midID := registry.NewID("test", "mid") + mainID := registry.NewID("test", "main") + compiled := &code.CompiledMain{ + MainID: mainID, + Imports: map[registry.ID][]code.Import{ + midID: {{ID: baseID, Alias: "b"}}, + mainID: {{ID: midID, Alias: "m"}}, + }, + Dependencies: []code.CompiledProto{ + {Name: "base", Node: &code.Node{ID: baseID}, Proto: compileTestProto(t, `ran = (ran or 0) + 1 return { v = 1 }`)}, + {Name: "mid", Node: &code.Node{ID: midID}, Proto: compileTestProto(t, `return { v = b.v + 1, via_require = require("b").v }`)}, + }, + } + proc := newDependencyProcess(t, compiled, ` + return { main = function() + return m.v * 100 + require("m").via_require * 10 + (b == nil and 1 or 0) + end } + `, 0) + state := proc.State() + if state.GetGlobal("ran") != lua.LNil { + t.Fatal("library code must not run before the first step") + } + if err := proc.Init(frameContext(), "main", nil); err != nil { + t.Fatal(err) + } + var output process.StepOutput + if err := proc.Step(nil, &output); err != nil { + t.Fatal(err) + } + // m.v = 2, require("m").via_require = 1, b hidden from main = 1. + if v, _ := output.Result().Data().(lua.LValue); lua.LVAsNumber(v) != 211 { + t.Fatalf("expected 211, got %v", output.Result().Data()) + } +} + +// A library that never returns is preempted instead of blocking Init, and a +// later execution of a process with finished libraries does not run them again. +func TestDependencyLoopIsPreempted(t *testing.T) { + id := registry.NewID("test", "spin") + compiled := &code.CompiledMain{ + Dependencies: []code.CompiledProto{{Name: "spin", Node: &code.Node{ID: id}, Proto: compileTestProto(t, `while true do end`)}}, + } + proc := newDependencyProcess(t, compiled, `return { main = function() end }`, 100) + proc.EnablePreemption() + + inited := make(chan error, 1) + go func() { inited <- proc.Init(frameContext(), "main", nil) }() + select { + case err := <-inited: + if err != nil { + t.Fatal(err) + } + case <-time.After(5 * time.Second): + t.Fatal("Init blocked on library code") + } + var output process.StepOutput + if err := proc.Step(nil, &output); err != nil { + t.Fatal(err) + } + if output.Status() != process.StepPreempted { + t.Fatalf("expected the library loop to be preempted, got %v", output.Status()) + } +} + +func TestDependenciesRunOncePerProcess(t *testing.T) { + id := registry.NewID("test", "counter") + mainID := registry.NewID("test", "main") + compiled := &code.CompiledMain{ + MainID: mainID, + Imports: map[registry.ID][]code.Import{mainID: {{ID: id, Alias: "c"}}}, + Dependencies: []code.CompiledProto{{Name: "counter", Node: &code.Node{ID: id}, Proto: compileTestProto(t, + `runs = (runs or 0) + 1 return { runs = runs }`)}}, + } + proc := newDependencyProcess(t, compiled, `return { main = function() return c.runs end }`, 0) + for i := 0; i < 3; i++ { + if err := proc.Init(frameContext(), "main", nil); err != nil { + t.Fatal(err) + } + var output process.StepOutput + if err := proc.Step(nil, &output); err != nil { + t.Fatal(err) + } + if v, _ := output.Result().Data().(lua.LValue); lua.LVAsNumber(v) != 1 { + t.Fatalf("execution %d: expected the library to have run once, got %v", i, output.Result().Data()) + } + } +} + +func frameContext() context.Context { + ctx, _ := ctxapi.OpenFrameContext(context.Background()) + return ctx +} + +// newDependencyProcess builds a process over script whose state is bound +// through the factory's isolation binder for compiled. +func newDependencyProcess(t *testing.T, compiled *code.CompiledMain, script string, tickBudget int64) *Process { + t.Helper() + binder := NewProcessFactory(nil).isolationBinder(compiled, newProcessConfig(), nil, nil, nil, nil) + budgets := luaapi.ExecutionBudgets{} + if tickBudget != 0 { + budgets = luaapi.ExecutionBudgets{TickBudget: tickBudget, TickBudgetSet: true} + } + proc := mustNewProcess(t, + WithModuleBinder(wrapBinder(func(l *lua.LState) { lua.OpenErrors(l) })), + WithModuleBinder(binder), + WithScript(script, "main.lua"), + WithProcessExecutionBudgets(budgets), + ) + t.Cleanup(proc.Close) + return proc +} + // --- helpers --- func compileTestProto(t *testing.T, source string) *lua.FunctionProto { diff --git a/runtime/lua/engine/process_isolation_test.go b/runtime/lua/engine/process_isolation_test.go index cef8ecba7..80b2bf4b0 100644 --- a/runtime/lua/engine/process_isolation_test.go +++ b/runtime/lua/engine/process_isolation_test.go @@ -76,6 +76,7 @@ func TestFactory_ImportIsolation_LibDepDoesNotLeak(t *testing.T) { if err != nil { t.Fatalf("factory() failed: %v", err) } + runEntry(t, proc.(*Process)) state := proc.(*Process).State() env := state.Env @@ -158,6 +159,7 @@ func TestFactory_ImportIsolation_OverlayLibrary(t *testing.T) { if err != nil { t.Fatalf("factory() failed: %v", err) } + runEntry(t, proc.(*Process)) state := proc.(*Process).State() env := state.Env @@ -230,6 +232,7 @@ func TestFactory_ImportIsolation_ExplicitGlobalExportIsVisibleWithoutImportLeak( defer procRaw.Close() luaProc := procRaw.(*Process) + runEntry(t, luaProc) state := luaProc.State() if got := state.GetGlobal("public_export"); got.String() != "visible" { @@ -316,3 +319,16 @@ func TestFactory_ImportIsolation_DynamicDSLGlobalOverridesImportAlias(t *testing t.Fatalf("migration DSL leaked into _G after define cleanup: %v", v) } } + +// runEntry runs the process's first step, which initializes its libraries. +func runEntry(t *testing.T, proc *Process) { + t.Helper() + ctx, _ := ctxapi.OpenFrameContext(context.Background()) + if err := proc.Init(ctx, "main", nil); err != nil { + t.Fatalf("Init failed: %v", err) + } + var output process.StepOutput + if err := proc.Step(nil, &output); err != nil { + t.Fatalf("Step failed: %v", err) + } +} From 7490d482213010509aaa3e712d92f2ac15478cb2 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Fri, 2 Oct 2026 21:18:54 -0400 Subject: [PATCH 07/50] chore(deps): go-lua with second-pass preemption fixes --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 3a0362042..a26597b61 100644 --- a/go.mod +++ b/go.mod @@ -69,7 +69,7 @@ require ( github.com/tree-sitter/tree-sitter-php v0.25.0 github.com/tree-sitter/tree-sitter-python v0.25.0 github.com/tree-sitter/tree-sitter-typescript v0.23.2 - github.com/wippyai/go-lua v1.6.4-0.20261002221722-a0cd38afeb7d + github.com/wippyai/go-lua v1.6.4-0.20261003005521-47a27bd56cfd github.com/wippyai/module-registry-proto-go v0.0.2-0.20260908140534-f6e2910c835f github.com/wippyai/tree-sitter-markdown v0.0.3 github.com/wippyai/tree-sitter-sql v0.0.4 diff --git a/go.sum b/go.sum index d783436bf..9bc2b08ff 100644 --- a/go.sum +++ b/go.sum @@ -546,6 +546,8 @@ github.com/wippyai/go-lua v1.6.2 h1:SJRjjW0XjDTVbFmPtjs3NHoHz1UgNdXMFJlCPg6aTQM= github.com/wippyai/go-lua v1.6.2/go.mod h1:cwD+390gJgx9cs+Zamby/lch5csqPV9RN1aDEfx/27M= github.com/wippyai/go-lua v1.6.4-0.20261002221722-a0cd38afeb7d h1:ZJjb1KcC83wKn4ndVPXJ1r2OJoZZneIuKMPxwwXRJYE= github.com/wippyai/go-lua v1.6.4-0.20261002221722-a0cd38afeb7d/go.mod h1:cwD+390gJgx9cs+Zamby/lch5csqPV9RN1aDEfx/27M= +github.com/wippyai/go-lua v1.6.4-0.20261003005521-47a27bd56cfd h1:vmuy4HscyBNOQs9DI75/yEkba4kQWhFkIv5YZkU3Dd8= +github.com/wippyai/go-lua v1.6.4-0.20261003005521-47a27bd56cfd/go.mod h1:cwD+390gJgx9cs+Zamby/lch5csqPV9RN1aDEfx/27M= github.com/wippyai/module-registry-proto-go v0.0.2-0.20260908140534-f6e2910c835f h1:AoKS058roqnsHXaJ8Gww7ODz14qJqOK5QBeoALvcq6I= github.com/wippyai/module-registry-proto-go v0.0.2-0.20260908140534-f6e2910c835f/go.mod h1:p4ihYQKRQuqRVLxaH1YL0IEnkz4zKdyfVADOwvA5Tno= github.com/wippyai/tree-sitter-markdown v0.0.3 h1:u6e53+hzPSS848Xhm+I48SwtvWQHrC21wDrogdwCwuc= From 1364424f7eb89b933fc9f114e3dec848434a65da Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Fri, 2 Oct 2026 21:28:52 -0400 Subject: [PATCH 08/50] fix(lua): run pending initializers in SyncExecute SyncExecute initializes the process's library chunks, in order and with the same result handling, before it calls the main chunk. --- runtime/lua/engine/initializer.go | 24 +++++++++++++ runtime/lua/engine/process.go | 4 +++ runtime/lua/engine/process_factory_test.go | 41 ++++++++++++++++++++++ 3 files changed, 69 insertions(+) diff --git a/runtime/lua/engine/initializer.go b/runtime/lua/engine/initializer.go index e91f27362..dd0d63de3 100644 --- a/runtime/lua/engine/initializer.go +++ b/runtime/lua/engine/initializer.go @@ -84,3 +84,27 @@ func (list *initializerList) advance(l *lua.LState) int { l.Push(list.items[list.done].Fn) return 1 } + +// runSync runs the incomplete initializers to completion on l's main thread, +// in order. It is the synchronous counterpart of the bootstrap loop for +// callers that execute without the scheduler; the code it runs is not +// preemptible. +func (list *initializerList) runSync(l *lua.LState) error { + if !list.pending() { + return nil + } + list.begin() + for list.done < len(list.items) { + item := list.items[list.done] + if err := l.CallByParam(lua.P{Fn: item.Fn, NRet: 1, Protect: true}); err != nil { + return err + } + result := l.Get(-1) + l.Pop(1) + list.done++ + if item.Done != nil { + item.Done(result) + } + } + return nil +} diff --git a/runtime/lua/engine/process.go b/runtime/lua/engine/process.go index 8c79473f9..a5b8b3692 100644 --- a/runtime/lua/engine/process.go +++ b/runtime/lua/engine/process.go @@ -2087,6 +2087,10 @@ func (p *Process) SyncExecute(ctx context.Context, args ...lua.LValue) (lua.LVal p.state.SetContext(ctx) + if err := initializersOf(p.state).runSync(p.state); err != nil { + return lua.LNil, toAPIError(err) + } + // Load function from proto fn := p.state.LoadProto(p.proto) diff --git a/runtime/lua/engine/process_factory_test.go b/runtime/lua/engine/process_factory_test.go index 431e41a88..3c0e9dc60 100644 --- a/runtime/lua/engine/process_factory_test.go +++ b/runtime/lua/engine/process_factory_test.go @@ -409,3 +409,44 @@ func compileTestProto(t *testing.T, source string) *lua.FunctionProto { require.NoError(t, err) return fn.Proto } + +// SyncExecute initializes the process's libraries before it runs the chunk. +func TestSyncExecuteRunsPendingInitializers(t *testing.T) { + id := registry.NewID("test", "c") + mainID := registry.NewID("test", "main") + compiled := &code.CompiledMain{ + MainID: mainID, + Imports: map[registry.ID][]code.Import{mainID: {{ID: id, Alias: "c"}}}, + Dependencies: []code.CompiledProto{{Name: "c", Node: &code.Node{ID: id}, Proto: compileTestProto(t, + `_G.runs = (_G.runs or 0) + 1 return { v = 42 }`)}}, + } + binder := NewProcessFactory(nil).isolationBinder(compiled, newProcessConfig(), nil, nil, nil, nil) + proc := mustNewProcess(t, WithModuleBinder(binder), WithProto(compileTestProto(t, `return c.v + runs`))) + t.Cleanup(proc.Close) + for i := 0; i < 2; i++ { + got, err := proc.SyncExecute(context.Background()) + if err != nil { + t.Fatal(err) + } + if lua.LVAsNumber(got) != 43 { + t.Fatalf("run %d: expected 43, got %v", i, got) + } + } +} + +func TestSyncExecuteInitializerErrorKeepsKind(t *testing.T) { + id := registry.NewID("test", "c") + compiled := &code.CompiledMain{ + Dependencies: []code.CompiledProto{{Name: "c", Node: &code.Node{ID: id}, Proto: compileTestProto(t, + `error(errors.new({message="bad dependency", kind=errors.INVALID, retryable=false}))`)}}, + } + binder := NewProcessFactory(nil).isolationBinder(compiled, newProcessConfig(), nil, nil, nil, nil) + proc := mustNewProcess(t, + WithModuleBinder(wrapBinder(func(l *lua.LState) { lua.OpenErrors(l) })), + WithModuleBinder(binder), WithProto(compileTestProto(t, `return 1`))) + t.Cleanup(proc.Close) + _, err := proc.SyncExecute(context.Background()) + if err == nil || apierror.BuildChain(err).Root().Kind != string(apierror.Invalid) { + t.Fatalf("expected an Invalid error from the library, got %v", err) + } +} From 2ee79a0718c87fb6dd3283235f2a554010e4cde3 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Fri, 2 Oct 2026 21:28:52 -0400 Subject: [PATCH 09/50] fix(lua): keep declared imports ahead of library global exports A chunk environment decides which aliases already exist in the base when the chunk first looks up a global, after its dependencies have run, so a library's _G export under an alias name no longer overrides the import while helpers installed later still resolve. --- runtime/lua/engine/process_factory.go | 20 ++++++++- runtime/lua/engine/process_factory_test.go | 51 ++++++++++++++++++++++ 2 files changed, 69 insertions(+), 2 deletions(-) diff --git a/runtime/lua/engine/process_factory.go b/runtime/lua/engine/process_factory.go index 8b06bf5df..ef5540312 100644 --- a/runtime/lua/engine/process_factory.go +++ b/runtime/lua/engine/process_factory.go @@ -298,14 +298,28 @@ func buildChunkEnv(l *lua.LState, base *lua.LTable, imports []code.Import, value // imports resolve at lookup time. Imports sharing an alias resolve to the // last one that has a value. aliasIDs := make(map[string][]registry.ID, len(imports)) - baseHadAlias := make(map[string]bool, len(imports)) for _, imp := range imports { alias := imp.Alias if alias == "" { alias = imp.ID.Name } aliasIDs[alias] = append(aliasIDs[alias], imp.ID) - baseHadAlias[alias] = base.RawGetString(alias) != lua.LNil + } + + // An alias that already exists in the base when the chunk first looks up a + // global is an import: the declared import wins over it. The chunk's own + // dependencies have run by then, so a library's explicit _G export under an + // alias name does not override the import, while helpers installed + // afterwards resolve through the base. + var baseHadAlias map[string]bool + snapshot := func() { + if baseHadAlias != nil { + return + } + baseHadAlias = make(map[string]bool, len(aliasIDs)) + for alias := range aliasIDs { + baseHadAlias[alias] = base.RawGetString(alias) != lua.LNil + } } resolve := func(alias string) (lua.LValue, bool) { ids := aliasIDs[alias] @@ -319,6 +333,7 @@ func buildChunkEnv(l *lua.LState, base *lua.LTable, imports []code.Import, value mt := l.NewTable() mt.RawSetString("__index", l.NewFunction(func(s *lua.LState) int { + snapshot() name := s.CheckString(2) baseValue := base.RawGetString(name) if baseValue != lua.LNil && !baseHadAlias[name] { @@ -342,6 +357,7 @@ func buildChunkEnv(l *lua.LState, base *lua.LTable, imports []code.Import, value l.SetMetatable(env, mt) env.RawSetString("require", l.NewFunction(func(s *lua.LState) int { + snapshot() name := s.CheckString(1) if v, ok := resolve(name); ok { s.Push(v) diff --git a/runtime/lua/engine/process_factory_test.go b/runtime/lua/engine/process_factory_test.go index 3c0e9dc60..cab9002d1 100644 --- a/runtime/lua/engine/process_factory_test.go +++ b/runtime/lua/engine/process_factory_test.go @@ -410,6 +410,43 @@ func compileTestProto(t *testing.T, source string) *lua.FunctionProto { return fn.Proto } +// A library that assigns _G under the alias it is imported as does not +// override the declared import: the alias resolves the same through the +// environment and through require. +func TestLibraryGlobalExportDoesNotOverrideImportAlias(t *testing.T) { + id := registry.NewID("test", "c") + mainID := registry.NewID("test", "main") + compiled := &code.CompiledMain{ + MainID: mainID, + Imports: map[registry.ID][]code.Import{mainID: {{ID: id, Alias: "c"}}}, + Dependencies: []code.CompiledProto{{Name: "c", Node: &code.Node{ID: id}, Proto: compileTestProto(t, + `_G.c = { v = 99 } _G.other = 7 return { v = 42 }`)}}, + } + proc := newDependencyProcess(t, compiled, ` + return { main = function() return c.v * 1000 + require("c").v + other end } + `, 0) + runProcess(t, proc, 42049) +} + +// A global installed after the environment is in use stays visible even when +// it shares a name with an import alias. +func TestDynamicGlobalOverridesImportAliasAfterInitialization(t *testing.T) { + id := registry.NewID("test", "dsl") + mainID := registry.NewID("test", "main") + compiled := &code.CompiledMain{ + MainID: mainID, + Imports: map[registry.ID][]code.Import{mainID: {{ID: id, Alias: "dsl"}}}, + Dependencies: []code.CompiledProto{{Name: "dsl", Node: &code.Node{ID: id}, Proto: compileTestProto(t, + `return { with = function(fn) _G.dsl = function() return 5 end local r = fn() _G.dsl = nil return r end }`)}}, + } + proc := newDependencyProcess(t, compiled, ` + return { main = function() + return require("dsl").with(function() return dsl() end) + end } + `, 0) + runProcess(t, proc, 5) +} + // SyncExecute initializes the process's libraries before it runs the chunk. func TestSyncExecuteRunsPendingInitializers(t *testing.T) { id := registry.NewID("test", "c") @@ -450,3 +487,17 @@ func TestSyncExecuteInitializerErrorKeepsKind(t *testing.T) { t.Fatalf("expected an Invalid error from the library, got %v", err) } } + +func runProcess(t *testing.T, proc *Process, want float64) { + t.Helper() + if err := proc.Init(frameContext(), "main", nil); err != nil { + t.Fatal(err) + } + var output process.StepOutput + if err := proc.Step(nil, &output); err != nil { + t.Fatal(err) + } + if v, _ := output.Result().Data().(lua.LValue); float64(lua.LVAsNumber(v)) != want { + t.Fatalf("expected %v, got %v", want, output.Result().Data()) + } +} From c699b3951347f3625c697f11d6241edb5fb2d71a Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Fri, 2 Oct 2026 21:40:39 -0400 Subject: [PATCH 10/50] fix(lua): fix import aliases when library initialization completes Each chunk environment seals its set of import aliases against the globals as they are once every library has initialized, so a later sibling library's _G export cannot override an earlier library's import regardless of when that library first looked up a global. --- runtime/lua/engine/initializer.go | 25 +++++++++++++ runtime/lua/engine/process_factory.go | 42 +++++++++++++++------- runtime/lua/engine/process_factory_test.go | 29 +++++++++++++++ 3 files changed, 83 insertions(+), 13 deletions(-) diff --git a/runtime/lua/engine/initializer.go b/runtime/lua/engine/initializer.go index dd0d63de3..5becfef36 100644 --- a/runtime/lua/engine/initializer.go +++ b/runtime/lua/engine/initializer.go @@ -43,6 +43,8 @@ func DeferInitializer(l *lua.LState, init Initializer) { // It implements lua.LValue so the state's registry can hold it. type initializerList struct { items []Initializer + // onComplete callbacks run once, when the last initializer completes. + onComplete []func() // done counts completed initializers; started reports that advance has // handed out items[done] in the current run. done int @@ -52,6 +54,18 @@ type initializerList struct { func (*initializerList) String() string { return "" } func (*initializerList) Type() lua.LValueType { return lua.LTUserData } +// OnInitialized registers fn to run when l's deferred initializers have all +// completed, before the entry chunk runs. It does not run if none are +// registered with DeferInitializer. +func OnInitialized(l *lua.LState, fn func()) { + list := initializersOf(l) + if list == nil { + list = &initializerList{} + l.G.Registry.RawSetString(initializersRegistryKey, list) + } + list.onComplete = append(list.onComplete, fn) +} + func initializersOf(l *lua.LState) *initializerList { list, _ := l.G.Registry.RawGetString(initializersRegistryKey).(*initializerList) return list @@ -76,6 +90,7 @@ func (list *initializerList) advance(l *lua.LState) int { if item.Done != nil { item.Done(l.Get(1)) } + list.finishIfDone() } list.started = list.done < len(list.items) if !list.started { @@ -105,6 +120,16 @@ func (list *initializerList) runSync(l *lua.LState) error { if item.Done != nil { item.Done(result) } + list.finishIfDone() } return nil } + +func (list *initializerList) finishIfDone() { + if list.done < len(list.items) { + return + } + for _, fn := range list.onComplete { + fn() + } +} diff --git a/runtime/lua/engine/process_factory.go b/runtime/lua/engine/process_factory.go index ef5540312..df838cc81 100644 --- a/runtime/lua/engine/process_factory.go +++ b/runtime/lua/engine/process_factory.go @@ -221,6 +221,7 @@ func (f *ProcessFactory) isolationBinder( base := l.Get(lua.GlobalsIndex).(*lua.LTable) valueByNode := make(map[registry.ID]lua.LValue, len(compiled.Dependencies)) processed := make(map[registry.ID]struct{}, len(compiled.Dependencies)) + var seals []func() // Dependencies are ordered deepest-first, so a library's imports are // resolved before the library itself runs. @@ -265,7 +266,8 @@ func (f *ProcessFactory) isolationBinder( } if dep.Proto != nil { - env := buildChunkEnv(l, base, compiled.Imports[id], valueByNode) + env, seal := buildChunkEnv(l, base, compiled.Imports[id], valueByNode) + seals = append(seals, seal) fn := l.LoadProto(dep.Proto) fn.Env = env DeferInitializer(l, Initializer{ @@ -277,7 +279,17 @@ func (f *ProcessFactory) isolationBinder( // The entrypoint runs under its own scoped environment. Assigning it as // the state environment makes LoadProto stamp the main chunk with it. - l.Env = buildChunkEnv(l, base, compiled.Imports[compiled.MainID], valueByNode) + env, seal := buildChunkEnv(l, base, compiled.Imports[compiled.MainID], valueByNode) + l.Env = env + seals = append(seals, seal) + + // Once every library has initialized, each environment fixes which + // aliases are imports against the globals as they then are. + OnInitialized(l, func() { + for _, seal := range seals { + seal() + } + }) return nil } } @@ -291,7 +303,7 @@ func (f *ProcessFactory) isolationBinder( // that global did not exist in the base environment at chunk creation time. This // preserves the old shared-_G migration DSL behavior without leaking declared // imports into _G. Undeclared modules resolve to nil and require fails closed. -func buildChunkEnv(l *lua.LState, base *lua.LTable, imports []code.Import, valueByNode map[registry.ID]lua.LValue) *lua.LTable { +func buildChunkEnv(l *lua.LState, base *lua.LTable, imports []code.Import, valueByNode map[registry.ID]lua.LValue) (*lua.LTable, func()) { env := l.NewTable() // Library values appear in valueByNode as their initializers complete, so @@ -306,21 +318,25 @@ func buildChunkEnv(l *lua.LState, base *lua.LTable, imports []code.Import, value aliasIDs[alias] = append(aliasIDs[alias], imp.ID) } - // An alias that already exists in the base when the chunk first looks up a - // global is an import: the declared import wins over it. The chunk's own - // dependencies have run by then, so a library's explicit _G export under an - // alias name does not override the import, while helpers installed - // afterwards resolve through the base. + // An alias that already exists in the base is an import: the declared + // import wins over it, so a library's explicit _G export under an alias + // name does not override the import, while helpers installed later resolve + // through the base. The returned seal fixes the set against the current + // globals; it is called when library initialization completes. Lookups + // before that, which only library top-level code makes, fix it on first + // use, with the chunk's own dependencies already run. var baseHadAlias map[string]bool - snapshot := func() { - if baseHadAlias != nil { - return - } + seal := func() { baseHadAlias = make(map[string]bool, len(aliasIDs)) for alias := range aliasIDs { baseHadAlias[alias] = base.RawGetString(alias) != lua.LNil } } + snapshot := func() { + if baseHadAlias == nil { + seal() + } + } resolve := func(alias string) (lua.LValue, bool) { ids := aliasIDs[alias] for i := len(ids) - 1; i >= 0; i-- { @@ -373,7 +389,7 @@ func buildChunkEnv(l *lua.LState, base *lua.LTable, imports []code.Import, value s.RaiseError("module '%s' not found", name) return 0 })) - return env + return env, seal } // Helper functions diff --git a/runtime/lua/engine/process_factory_test.go b/runtime/lua/engine/process_factory_test.go index cab9002d1..e05fb706e 100644 --- a/runtime/lua/engine/process_factory_test.go +++ b/runtime/lua/engine/process_factory_test.go @@ -501,3 +501,32 @@ func runProcess(t *testing.T, proc *Process, want float64) { t.Fatalf("expected %v, got %v", want, output.Result().Data()) } } + +// Every chunk environment decides its import aliases when library +// initialization completes: a sibling library's _G export under an alias name +// does not override the import in an earlier library, whether or not that +// library looked up a global while it initialized. +func TestSiblingGlobalExportDoesNotOverrideEarlierLibraryImport(t *testing.T) { + for name, aSource := range map[string]string{ + "no top-level lookup": `return { get = function() return c.v end }`, + "top-level global read": `local _ = tostring return { get = function() return c.v end }`, + } { + t.Run(name, func(t *testing.T) { + aID, bID, cID, mainID := registry.NewID("t", "a"), registry.NewID("t", "b"), registry.NewID("t", "c"), registry.NewID("t", "main") + compiled := &code.CompiledMain{ + MainID: mainID, + Imports: map[registry.ID][]code.Import{ + aID: {{ID: cID, Alias: "c"}}, + mainID: {{ID: aID, Alias: "a"}, {ID: bID, Alias: "b"}}, + }, + Dependencies: []code.CompiledProto{ + {Name: "c", Node: &code.Node{ID: cID}, Proto: compileTestProto(t, `return { v = 42 }`)}, + {Name: "a", Node: &code.Node{ID: aID}, Proto: compileTestProto(t, aSource)}, + {Name: "b", Node: &code.Node{ID: bID}, Proto: compileTestProto(t, `_G.c = { v = 99 } return {}`)}, + }, + } + proc := newDependencyProcess(t, compiled, `return { main = function() return a.get() end }`, 0) + runProcess(t, proc, 42) + }) + } +} From f4e0727b9d31be635ad2d5f94f0dd2b6fd3e2927 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Fri, 2 Oct 2026 22:04:09 -0400 Subject: [PATCH 11/50] fix(actor): Stop cancels stuck processes through their published incarnation On timeout Stop read each processor's cancel field while Submit could still be assigning it after publishing the processor, and while completion could be clearing it for pooling. It now cancels through the processor's atomically published signal reference, as Terminate does; the cancellation still wakes parked processes through the wake registered at submit. --- system/scheduler/actor/scheduler.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/system/scheduler/actor/scheduler.go b/system/scheduler/actor/scheduler.go index 68053bf6f..8473a58ab 100644 --- a/system/scheduler/actor/scheduler.go +++ b/system/scheduler/actor/scheduler.go @@ -202,11 +202,13 @@ func (s *Scheduler) Stop(ctx context.Context) { case <-s.drainCh: // All processes completed gracefully case <-waitCtx.Done(): - // Timeout - cancel all process contexts to unblock stuck processes + // Timeout - cancel all process contexts to unblock stuck processes. + // A processor may be starting or completing concurrently, so act + // only through its published incarnation, as Terminate does. s.byPID.Range(func(_, value any) bool { proc := value.(*Processor) - if proc.cancel != nil { - proc.cancel() + if ref := proc.sig.Load(); ref != nil { + ref.terminate() } proc.queue.Close() return true From 3c871c787956c0decb0f01d8781a3a7a63eaf2ba Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Fri, 2 Oct 2026 22:04:29 -0400 Subject: [PATCH 12/50] chore(deps): go-lua with third-pass preemption fixes --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index a26597b61..92e6aa1e4 100644 --- a/go.mod +++ b/go.mod @@ -69,7 +69,7 @@ require ( github.com/tree-sitter/tree-sitter-php v0.25.0 github.com/tree-sitter/tree-sitter-python v0.25.0 github.com/tree-sitter/tree-sitter-typescript v0.23.2 - github.com/wippyai/go-lua v1.6.4-0.20261003005521-47a27bd56cfd + github.com/wippyai/go-lua v1.6.4-0.20261003012915-c14848bbdb76 github.com/wippyai/module-registry-proto-go v0.0.2-0.20260908140534-f6e2910c835f github.com/wippyai/tree-sitter-markdown v0.0.3 github.com/wippyai/tree-sitter-sql v0.0.4 diff --git a/go.sum b/go.sum index 9bc2b08ff..280da1f11 100644 --- a/go.sum +++ b/go.sum @@ -548,6 +548,8 @@ github.com/wippyai/go-lua v1.6.4-0.20261002221722-a0cd38afeb7d h1:ZJjb1KcC83wKn4 github.com/wippyai/go-lua v1.6.4-0.20261002221722-a0cd38afeb7d/go.mod h1:cwD+390gJgx9cs+Zamby/lch5csqPV9RN1aDEfx/27M= github.com/wippyai/go-lua v1.6.4-0.20261003005521-47a27bd56cfd h1:vmuy4HscyBNOQs9DI75/yEkba4kQWhFkIv5YZkU3Dd8= github.com/wippyai/go-lua v1.6.4-0.20261003005521-47a27bd56cfd/go.mod h1:cwD+390gJgx9cs+Zamby/lch5csqPV9RN1aDEfx/27M= +github.com/wippyai/go-lua v1.6.4-0.20261003012915-c14848bbdb76 h1:jTTXVVQ9q3HK2SUnFnzQj2SCInmKFTKTlE4XDdZ3WbQ= +github.com/wippyai/go-lua v1.6.4-0.20261003012915-c14848bbdb76/go.mod h1:cwD+390gJgx9cs+Zamby/lch5csqPV9RN1aDEfx/27M= github.com/wippyai/module-registry-proto-go v0.0.2-0.20260908140534-f6e2910c835f h1:AoKS058roqnsHXaJ8Gww7ODz14qJqOK5QBeoALvcq6I= github.com/wippyai/module-registry-proto-go v0.0.2-0.20260908140534-f6e2910c835f/go.mod h1:p4ihYQKRQuqRVLxaH1YL0IEnkz4zKdyfVADOwvA5Tno= github.com/wippyai/tree-sitter-markdown v0.0.3 h1:u6e53+hzPSS848Xhm+I48SwtvWQHrC21wDrogdwCwuc= From f0cdb0aa5cfdc1133155a27c4ee87d7f5d316bb5 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Fri, 2 Oct 2026 22:48:33 -0400 Subject: [PATCH 13/50] fix(actor): close a process exactly once on failed upgrade and rejected admission Completion owns closing a non-pooled process and the submitter owns a process Submit rejects. The worker closed the process before completing it on every failed upgrade, and Submit closed it on OnStart rejection while the caller closed it again; a Lua process returns itself to its pool on each Close, so the duplicate put could hand one process to two users. --- .../scheduler/actor/close_ownership_test.go | 110 ++++++++++++++++++ system/scheduler/actor/scheduler.go | 1 - system/scheduler/actor/worker.go | 8 -- 3 files changed, 110 insertions(+), 9 deletions(-) create mode 100644 system/scheduler/actor/close_ownership_test.go diff --git a/system/scheduler/actor/close_ownership_test.go b/system/scheduler/actor/close_ownership_test.go new file mode 100644 index 000000000..3517b3c58 --- /dev/null +++ b/system/scheduler/actor/close_ownership_test.go @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: MPL-2.0 + +package actor + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + ctxapi "github.com/wippyai/runtime/api/context" + "github.com/wippyai/runtime/api/payload" + pidapi "github.com/wippyai/runtime/api/pid" + "github.com/wippyai/runtime/api/process" + "github.com/wippyai/runtime/api/registry" + apiruntime "github.com/wippyai/runtime/api/runtime" +) + +// closeCountingProcess counts Close calls. +type closeCountingProcess struct { + upgradeSource + closed atomic.Int64 +} + +func (p *closeCountingProcess) Close() { p.closed.Add(1) } + +// A process whose admission fails stays owned by the submitter, which closes it. +func TestSubmitRejectedByLifecycleDoesNotCloseProcess(t *testing.T) { + sched := newPreemptTestScheduler(1, &testLifecycle{}) + sched.lifecycle = &rejectingLifecycle2{} + sched.Start() + defer testStopScheduler(sched) + + p := &closeCountingProcess{} + if _, err := sched.Submit(context.Background(), pidapi.PID{UniqID: "r"}, p, "", nil); err == nil { + t.Fatal("expected rejection") + } + if got := p.closed.Load(); got != 0 { + t.Fatalf("scheduler closed a process it did not admit (%d)", got) + } +} + +// Every failed upgrade closes the process exactly once. +func TestFailedUpgradeClosesProcessOnce(t *testing.T) { + cases := map[string]func(*closeCountingProcess) (process.Factory, *upgradeFailure){ + "create": func(*closeCountingProcess) (process.Factory, *upgradeFailure) { + return &mockFactory{createFunc: func(registry.ID) (process.Process, *process.Meta, error) { + return nil, nil, errors.New("create failed") + }}, nil + }, + "init": func(*closeCountingProcess) (process.Factory, *upgradeFailure) { + target := &upgradeFailure{} + return &mockFactory{createFunc: func(registry.ID) (process.Process, *process.Meta, error) { + return target, &process.Meta{Method: "main"}, nil + }}, target + }, + } + for name, setup := range cases { + t.Run(name, func(t *testing.T) { + done := make(chan struct{}) + lc := &testLifecycle{onComplete: func(context.Context, pidapi.PID, *apiruntime.Result) { close(done) }} + sched := newPreemptTestScheduler(1, lc) + sched.Start() + defer testStopScheduler(sched) + + source := &closeCountingProcess{upgradeSource: upgradeSource{UpgradeProcess: UpgradeProcess{ + upgradeReq: &process.UpgradeRequest{Source: registry.NewID("app", "next")}}}} + factory, target := setup(source) + appCtx := ctxapi.WithAppContext(context.Background(), ctxapi.NewAppContext()) + process.WithFactory(appCtx, factory) + frameCtx, fc := ctxapi.OpenFrameContext(appCtx) + fc.Seal() + if _, err := sched.Submit(frameCtx, pidapi.PID{UniqID: "u"}, source, "", nil); err != nil { + t.Fatal(err) + } + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("actor did not complete") + } + time.Sleep(50 * time.Millisecond) + closes := source.closed.Load() + if target != nil { + closes += target.closed.Load() + } + wantTotal := int64(1) + if target != nil { + wantTotal = 2 + } + if closes != wantTotal { + t.Fatalf("expected %d closes across incarnations, got %d", wantTotal, closes) + } + if target != nil && target.closed.Load() != 1 { + t.Fatalf("replacement closed %d times", target.closed.Load()) + } + }) + } +} + +// upgradeFailure is a replacement whose Init fails. +type upgradeFailure struct { + upgradeTarget + closed atomic.Int64 +} + +func (p *upgradeFailure) Init(context.Context, string, payload.Payloads) error { + return errors.New("init failed") +} +func (p *upgradeFailure) Close() { p.closed.Add(1) } diff --git a/system/scheduler/actor/scheduler.go b/system/scheduler/actor/scheduler.go index 8473a58ab..5e2bc7bb1 100644 --- a/system/scheduler/actor/scheduler.go +++ b/system/scheduler/actor/scheduler.go @@ -361,7 +361,6 @@ func (s *Scheduler) Submit(ctx context.Context, pid pid.PID, p process.Process, s.byQueue.Delete(proc.queue) s.processorCount.Add(-1) cancel() - p.Close() releaseProcessor(proc) return nil, err } diff --git a/system/scheduler/actor/worker.go b/system/scheduler/actor/worker.go index 7f8b5bf2e..7bbb35041 100644 --- a/system/scheduler/actor/worker.go +++ b/system/scheduler/actor/worker.go @@ -475,7 +475,6 @@ func (w *Worker) executeOne(proc *Processor) { case process.StepUpgrade: req := proc.output.Upgrade() if req == nil { - proc.Process.Close() if !proc.casState(StateRunning, StateComplete) { return } @@ -486,7 +485,6 @@ func (w *Worker) executeOne(proc *Processor) { factory := process.GetFactory(proc.ctx) if factory == nil { - proc.Process.Close() if !proc.casState(StateRunning, StateComplete) { return } @@ -501,7 +499,6 @@ func (w *Worker) executeOne(proc *Processor) { var ok bool source, ok = runtime.GetFrameID(proc.ctx) if !ok { - proc.Process.Close() if !proc.casState(StateRunning, StateComplete) { return } @@ -514,7 +511,6 @@ func (w *Worker) executeOne(proc *Processor) { // Create new process newProc, meta, err := factory.Create(source) if err != nil { - proc.Process.Close() if !proc.casState(StateRunning, StateComplete) { return } @@ -547,7 +543,6 @@ func (w *Worker) executeOne(proc *Processor) { // the process by its NEW definition (used by ListProcesses and OUTDATED // notification), not the pre-upgrade source. if err := runtime.SetFrameID(upgradeCtx, source); err != nil { - proc.Process.Close() if !proc.casState(StateRunning, StateComplete) { return } @@ -559,7 +554,6 @@ func (w *Worker) executeOne(proc *Processor) { // current definition, so the replacement frame carries them over. if opts := runtime.GetFrameLifecycleOptions(proc.ctx); opts != nil { if err := runtime.SetFrameLifecycleOptions(upgradeCtx, opts); err != nil { - proc.Process.Close() if !proc.casState(StateRunning, StateComplete) { return } @@ -570,7 +564,6 @@ func (w *Worker) executeOne(proc *Processor) { } if hasSelfPID { if err := runtime.SetFramePID(upgradeCtx, selfPID); err != nil { - proc.Process.Close() if !proc.casState(StateRunning, StateComplete) { return } @@ -580,7 +573,6 @@ func (w *Worker) executeOne(proc *Processor) { } } if err := newProc.Init(upgradeCtx, method, req.Input); err != nil { - proc.Process.Close() if !proc.casState(StateRunning, StateComplete) { return } From 0b7a7258a61a58df042b0f8310df4a1d83fb3c03 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Fri, 2 Oct 2026 22:52:07 -0400 Subject: [PATCH 14/50] fix(actor): bind yield completions to the dispatching incarnation Handlers received the pooled Processor as their receiver, and its CompleteYield read the current queue generation. A completion that outlived its actor and arrived after the slot was reused enqueued an event into, and woke, the new actor. Handlers now receive the queue's generation-bound YieldCompleter, and Processor no longer implements the receiver interface. --- .../scheduler/actor/close_ownership_test.go | 90 +++++++++++++++++++ system/scheduler/actor/edge_cases_test.go | 4 +- system/scheduler/actor/processor.go | 51 +++-------- system/scheduler/actor/scheduler.go | 2 + system/scheduler/actor/worker.go | 2 +- 5 files changed, 107 insertions(+), 42 deletions(-) diff --git a/system/scheduler/actor/close_ownership_test.go b/system/scheduler/actor/close_ownership_test.go index 3517b3c58..88c5b5aa7 100644 --- a/system/scheduler/actor/close_ownership_test.go +++ b/system/scheduler/actor/close_ownership_test.go @@ -5,6 +5,9 @@ package actor import ( "context" "errors" + "github.com/wippyai/runtime/api/dispatcher" + "github.com/wippyai/runtime/api/relay" + "github.com/wippyai/runtime/system/scheduler" "sync/atomic" "testing" "time" @@ -108,3 +111,90 @@ func (p *upgradeFailure) Init(context.Context, string, payload.Payloads) error { return errors.New("init failed") } func (p *upgradeFailure) Close() { p.closed.Add(1) } + +// waitingProcess yields once and then waits for the completion. +type waitingProcess struct { + steps atomic.Int64 +} + +func (*waitingProcess) Init(context.Context, string, payload.Payloads) error { return nil } +func (p *waitingProcess) Step(events []process.Event, out *process.StepOutput) error { + if p.steps.Add(1) == 1 { + out.Yield(YieldCmd{}, 7) + out.WaitForYields() + return nil + } + if len(events) > 0 { + out.Done(nil) + return nil + } + out.WaitForYields() + return nil +} +func (*waitingProcess) Send(*relay.Package) error { return nil } +func (*waitingProcess) Close() {} + +// A completion that arrives after its actor ended is dropped, even when the +// processor slot has been reused by another actor. +func TestStaleYieldCompletionDoesNotWakeReusedProcessor(t *testing.T) { + received := make(chan dispatcher.ResultReceiver, 1) + reg := scheduler.NewRegistry() + reg.Register(CmdYield, dispatcher.HandlerFunc(func(_ context.Context, _ dispatcher.Command, _ uint64, r dispatcher.ResultReceiver) error { + select { + case received <- r: + default: + } + return nil + })) + reg.Register(CmdComplete, CompleteHandler()) + done := make(chan struct{}, 64) + lc := &testLifecycle{onComplete: func(context.Context, pidapi.PID, *apiruntime.Result) { done <- struct{}{} }} + sched := NewScheduler(reg, WithWorkers(1), WithLifecycle(lc)) + sched.Start() + defer testStopScheduler(sched) + + ctxA, cancelA := context.WithCancel(context.Background()) + a := &waitingProcess{} + procA, err := sched.Submit(ctxA, pidapi.PID{UniqID: "a"}, a, "", nil) + if err != nil { + t.Fatal(err) + } + var stale dispatcher.ResultReceiver + select { + case stale = <-received: + case <-time.After(5 * time.Second): + t.Fatal("yield was not dispatched") + } + cancelA() + <-done + + for i := 0; i < 1000; i++ { + ctxB, cancelB := context.WithCancel(context.Background()) + b := &waitingProcess{} + procB, err := sched.Submit(ctxB, pidapi.PID{UniqID: "b"}, b, "", nil) + if err != nil { + t.Fatal(err) + } + for b.steps.Load() == 0 { + time.Sleep(time.Millisecond) + } + select { + case <-received: + case <-time.After(5 * time.Second): + t.Fatal("yield was not dispatched") + } + if procB == procA { + stale.CompleteYield(7, nil, nil) + time.Sleep(50 * time.Millisecond) + if got := b.steps.Load(); got != 1 { + t.Fatalf("stale completion woke the reused processor: %d steps", got) + } + cancelB() + <-done + return + } + cancelB() + <-done + } + t.Fatal("processor slot was never reused") +} diff --git a/system/scheduler/actor/edge_cases_test.go b/system/scheduler/actor/edge_cases_test.go index af343f681..b1644acfd 100644 --- a/system/scheduler/actor/edge_cases_test.go +++ b/system/scheduler/actor/edge_cases_test.go @@ -867,11 +867,9 @@ func TestCollectProcessStatsWithProvider(t *testing.T) { func TestWakeProcessorPath(t *testing.T) { reg := scheduler.NewRegistry() reg.Register(CmdYield, dispatcher.HandlerFunc(func(_ context.Context, _ dispatcher.Command, tag uint64, receiver dispatcher.ResultReceiver) error { - proc := receiver.(*Processor) - completer := proc.queue.NewYieldCompleter(proc.scheduler) go func() { time.Sleep(10 * time.Millisecond) - completer.CompleteYield(tag, nil, nil) + receiver.CompleteYield(tag, nil, nil) }() return nil })) diff --git a/system/scheduler/actor/processor.go b/system/scheduler/actor/processor.go index 10b97416c..f8e9e0056 100644 --- a/system/scheduler/actor/processor.go +++ b/system/scheduler/actor/processor.go @@ -76,14 +76,18 @@ const ( // - StateBlocked: no owner, CompleteYield can CAS to Ready and re-queue // - Worker uses CAS loop to atomically check wakeup and transition state type Processor struct { - ctx context.Context - Process process.Process - stats atomic.Pointer[attrs.Bag] - resultCh chan *runtime.Result - scheduler *Scheduler - cancel context.CancelFunc - queue *process.EventQueue - sig atomic.Pointer[signalRef] + ctx context.Context + Process process.Process + stats atomic.Pointer[attrs.Bag] + resultCh chan *runtime.Result + scheduler *Scheduler + cancel context.CancelFunc + queue *process.EventQueue + sig atomic.Pointer[signalRef] + // completer receives yield completions for the current incarnation. It is + // bound to the queue generation, so completions that outlive the + // incarnation do not reach a later user of this pooled processor. + completer *process.YieldCompleter inspector atomic.Pointer[inspectorRef] pid pid.PID output process.StepOutput @@ -188,36 +192,6 @@ func (p *Processor) finishDispatch() bool { } } -// CompleteYield implements dispatcher.ResultReceiver. -// Called by handlers to deliver yield completion. -// Thread-safe: can be called from any goroutine. -func (p *Processor) CompleteYield(tag uint64, data any, err error) { - if !p.queue.Push(process.Event{ - Type: process.EventYieldComplete, - Tag: tag, - Data: data, - Error: err, - }, p.gen.Load()) { - return - } - - // Try to transition Blocked→Ready and re-queue. - // If processor is Running (worker still owns it), set wakeup flag atomically. - // Worker will check wakeup after dispatch and re-queue if set. - if p.casState(StateBlocked, StateReady) { - sched := p.scheduler - if sched != nil { - sched.injectOrGlobal(p) - } - return - } - - // Failed to transition - processor might be Running. - // Try to set wakeup flag atomically. If state changed, that's fine - - // either worker already moved on or another CompleteYield already woke it. - p.setWakeup(StateRunning) -} - // StateName returns a human-readable name for the process state. func StateName(s ProcessState) string { switch s & stateMask { @@ -264,6 +238,7 @@ func releaseProcessor(p *Processor) { p.Process = nil p.ctx = nil p.cancel = nil + p.completer = nil p.scheduler = nil p.lastWorker.Store(noWorkerAffinity) p.resultCh = nil diff --git a/system/scheduler/actor/scheduler.go b/system/scheduler/actor/scheduler.go index 5e2bc7bb1..1c24402a3 100644 --- a/system/scheduler/actor/scheduler.go +++ b/system/scheduler/actor/scheduler.go @@ -347,6 +347,7 @@ func (s *Scheduler) Submit(ctx context.Context, pid pid.PID, p process.Process, proc.queue.SetAdmission(admission.EventAdmission()) } proc.gen.Store(proc.queue.Generation()) + proc.completer = proc.queue.NewYieldCompleter(s) proc.publishSignalRef(cancel) proc.publishInspectorRef() @@ -480,6 +481,7 @@ func (s *Scheduler) CreateProcessor(ctx context.Context, pid pid.PID, p process. // Reset queue for this execution and cache generation proc.queue.Reset() proc.gen.Store(proc.queue.Generation()) + proc.completer = proc.queue.NewYieldCompleter(s) proc.publishSignalRef(cancel) proc.publishInspectorRef() diff --git a/system/scheduler/actor/worker.go b/system/scheduler/actor/worker.go index 7bbb35041..1b2060deb 100644 --- a/system/scheduler/actor/worker.go +++ b/system/scheduler/actor/worker.go @@ -614,7 +614,7 @@ func (w *Worker) dispatchYields(ctx context.Context, proc *Processor, yields []p }) continue } - if err := handler.Handle(ctx, y.Cmd, y.Tag, proc); err != nil { + if err := handler.Handle(ctx, y.Cmd, y.Tag, proc.completer); err != nil { proc.queue.PushDirect(process.Event{ Type: process.EventYieldComplete, Tag: y.Tag, From a60d829139eff2c2fc4a0e88a8e1942921dfaa9a Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Fri, 2 Oct 2026 22:54:06 -0400 Subject: [PATCH 15/50] fix(lua): drop completed initializers and one-shot callbacks from the state The registry-held initializer list kept every completed Fn, Done and OnInitialized callback, retaining chunk environments and their captures for the life of the state. A completed entry is cleared and the callbacks are released after they run; failed initializers stay registered for retry. --- runtime/lua/engine/initializer.go | 28 ++++++----- .../lua/engine/initializer_release_test.go | 47 +++++++++++++++++++ 2 files changed, 62 insertions(+), 13 deletions(-) create mode 100644 runtime/lua/engine/initializer_release_test.go diff --git a/runtime/lua/engine/initializer.go b/runtime/lua/engine/initializer.go index 5becfef36..93a1a0a45 100644 --- a/runtime/lua/engine/initializer.go +++ b/runtime/lua/engine/initializer.go @@ -85,12 +85,7 @@ func (list *initializerList) begin() { // function to run, or nothing when all have run. func (list *initializerList) advance(l *lua.LState) int { if list.started { - item := list.items[list.done] - list.done++ - if item.Done != nil { - item.Done(l.Get(1)) - } - list.finishIfDone() + list.complete(l.Get(1)) } list.started = list.done < len(list.items) if !list.started { @@ -116,20 +111,27 @@ func (list *initializerList) runSync(l *lua.LState) error { } result := l.Get(-1) l.Pop(1) - list.done++ - if item.Done != nil { - item.Done(result) - } - list.finishIfDone() + list.complete(result) } return nil } -func (list *initializerList) finishIfDone() { +// complete records the result of the current initializer and drops it, so the +// state keeps neither the function nor its callback after the run. The last +// completion runs the one-shot callbacks and drops them as well. +func (list *initializerList) complete(result lua.LValue) { + item := list.items[list.done] + list.items[list.done] = Initializer{} + list.done++ + if item.Done != nil { + item.Done(result) + } if list.done < len(list.items) { return } - for _, fn := range list.onComplete { + callbacks := list.onComplete + list.onComplete = nil + for _, fn := range callbacks { fn() } } diff --git a/runtime/lua/engine/initializer_release_test.go b/runtime/lua/engine/initializer_release_test.go new file mode 100644 index 000000000..b1e26c017 --- /dev/null +++ b/runtime/lua/engine/initializer_release_test.go @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: MPL-2.0 + +package engine + +import ( + "runtime" + "sync/atomic" + "testing" + "time" + + lua "github.com/wippyai/go-lua" +) + +// Completed initializers and one-shot completion callbacks are not retained by +// the state that outlives them. +func TestCompletedInitializersAreReleased(t *testing.T) { + var collected atomic.Int64 + probe := func() *finalizerProbe { + p := &finalizerProbe{collected: &collected} + runtime.SetFinalizer(p, func(p *finalizerProbe) { p.collected.Add(1) }) + return p + } + // The binder creates the captures itself: a binder that closed over them + // would keep them alive through the process's factory. + proc := mustNewProcess(t, + WithModuleBinder(wrapBinder(func(l *lua.LState) { + initProbe, doneProbe, callbackProbe := probe(), probe(), probe() + DeferInitializer(l, Initializer{ + Fn: l.NewFunction(func(*lua.LState) int { _ = initProbe; return 0 }), + Done: func(lua.LValue) { _ = doneProbe }, + }) + OnInitialized(l, func() { _ = callbackProbe }) + })), + WithScript(`return { main = function() return 1 end }`, "m.lua")) + t.Cleanup(proc.Close) + + runProcess(t, proc, 1) + + deadline := time.Now().Add(5 * time.Second) + for collected.Load() < 3 && time.Now().Before(deadline) { + runtime.GC() + time.Sleep(10 * time.Millisecond) + } + if got := collected.Load(); got != 3 { + t.Fatalf("%d of 3 completed initializer captures were collected", got) + } +} From 9ee145b77a77e822a319ad2d7a7064c1d50644c4 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Fri, 2 Oct 2026 22:54:47 -0400 Subject: [PATCH 16/50] fix(lua): clear a closed task's result and resume buffers Pooled tasks truncated retBuf and resumeBuf without clearing, so the backing arrays kept the previous execution's Lua values reachable. --- runtime/lua/engine/task.go | 6 ++++++ runtime/lua/engine/task_test.go | 16 ++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/runtime/lua/engine/task.go b/runtime/lua/engine/task.go index 4935854fb..4c669dd48 100644 --- a/runtime/lua/engine/task.go +++ b/runtime/lua/engine/task.go @@ -171,6 +171,12 @@ func (t *Task) Close() { t.fn = nil t.Yielded = nil t.Resumed = nil + // The buffers outlive the task in the pool, so their backing arrays must + // not keep the last execution's values reachable. + clear(t.retBuf[:cap(t.retBuf)]) + t.retBuf = t.retBuf[:0] + clear(t.resumeBuf[:cap(t.resumeBuf)]) + t.resumeBuf = t.resumeBuf[:0] t.State = 0 taskPool.Put(t) } diff --git a/runtime/lua/engine/task_test.go b/runtime/lua/engine/task_test.go index fe1520ee1..3726903d3 100644 --- a/runtime/lua/engine/task_test.go +++ b/runtime/lua/engine/task_test.go @@ -218,3 +218,19 @@ func TestTask_PoolReuse(t *testing.T) { assert.Equal(t, lua.ResumeYield, task2.State) task2.Close() } + +// A closed task keeps no Lua values in its reusable buffers. +func TestTaskCloseClearsReusableBuffers(t *testing.T) { + thread := lua.NewState() + task := NewTask(thread, nil) + task.retBuf = append(task.retBuf, lua.LString("result"), lua.LString("result2")) + task.ResumeWith(lua.LString("resumed"), lua.LString("resumed2")) + task.Close() + + for i, v := range task.retBuf[:cap(task.retBuf)] { + require.Nil(t, v, "retBuf[%d]", i) + } + for i, v := range task.resumeBuf[:cap(task.resumeBuf)] { + require.Nil(t, v, "resumeBuf[%d]", i) + } +} From 2de740fafd573fd28ef936c7960ef11e8f8eec04 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Fri, 2 Oct 2026 22:54:47 -0400 Subject: [PATCH 17/50] fix(process): clear overflow yields when a step output resets Reset truncated the overflow slice without clearing it, so the pooled output kept references to the previous step's commands. --- api/process/process_test.go | 18 ++++++++++++++++++ api/process/step.go | 1 + 2 files changed, 19 insertions(+) diff --git a/api/process/process_test.go b/api/process/process_test.go index 3e2eacc1f..16afed8a8 100644 --- a/api/process/process_test.go +++ b/api/process/process_test.go @@ -511,3 +511,21 @@ func (g *stubPIDGenerator) Generate(host pid.HostID) pid.PID { result.Host = host return result } + +type overflowCmd struct{ id dispatcher.CommandID } + +func (c overflowCmd) CmdID() dispatcher.CommandID { return c.id } + +// A reset output keeps no command references in its overflow buffer. +func TestStepOutputResetClearsOverflowYields(t *testing.T) { + var out StepOutput + for i := 0; i < MaxYields+3; i++ { + out.Yield(overflowCmd{id: dispatcher.CommandID(i)}, uint64(i)) + } + out.Reset() + for i, y := range out.ext[:cap(out.ext)] { + if y.Cmd != nil { + t.Fatalf("overflow yield %d still references its command", i) + } + } +} diff --git a/api/process/step.go b/api/process/step.go index 093f996e1..1e85333d7 100644 --- a/api/process/step.go +++ b/api/process/step.go @@ -111,6 +111,7 @@ func (o *StepOutput) Preempt() { func (o *StepOutput) Reset() { o.buf[0] = Yield{} o.buf[1] = Yield{} + clear(o.ext) o.ext = o.ext[:0] o.count = 0 o.status = StepContinue From 510c05e9f089589d500111349a989f5f40de09cc Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Fri, 2 Oct 2026 22:54:48 -0400 Subject: [PATCH 18/50] test: measure goroutine, heap and reference release across preemption cycles --- runtime/lua/engine/preempt_leak_test.go | 309 ++++++++++++++++++++ system/scheduler/actor/preempt_leak_test.go | 125 ++++++++ 2 files changed, 434 insertions(+) create mode 100644 runtime/lua/engine/preempt_leak_test.go create mode 100644 system/scheduler/actor/preempt_leak_test.go diff --git a/runtime/lua/engine/preempt_leak_test.go b/runtime/lua/engine/preempt_leak_test.go new file mode 100644 index 000000000..1b7faa80e --- /dev/null +++ b/runtime/lua/engine/preempt_leak_test.go @@ -0,0 +1,309 @@ +// SPDX-License-Identifier: MPL-2.0 + +package engine + +import ( + "context" + "errors" + "fmt" + "reflect" + "runtime" + "sync" + "sync/atomic" + "testing" + "time" + + lua "github.com/wippyai/go-lua" + "github.com/wippyai/runtime/api/attrs" + ctxapi "github.com/wippyai/runtime/api/context" + "github.com/wippyai/runtime/api/process" + "github.com/wippyai/runtime/api/registry" + "github.com/wippyai/runtime/api/relay" + apiruntime "github.com/wippyai/runtime/api/runtime" + luaapi "github.com/wippyai/runtime/api/runtime/lua" + "github.com/wippyai/runtime/runtime/lua/code" + scheduler "github.com/wippyai/runtime/system/scheduler/actor" +) + +func heapInUse() uint64 { + runtime.GC() + runtime.GC() + var m runtime.MemStats + runtime.ReadMemStats(&m) + return m.HeapAlloc +} + +func settleGoroutines(want int) int { + n := 0 + for i := 0; i < 200; i++ { + runtime.GC() + if n = runtime.NumGoroutine(); n <= want { + return n + } + time.Sleep(10 * time.Millisecond) + } + return n +} + +func libraryCompiled(t *testing.T) *code.CompiledMain { + t.Helper() + baseID := registry.NewID("test", "base") + midID := registry.NewID("test", "mid") + mainID := registry.NewID("test", "main") + return &code.CompiledMain{ + MainID: mainID, + Imports: map[registry.ID][]code.Import{ + midID: {{ID: baseID, Alias: "b"}}, + mainID: {{ID: midID, Alias: "m"}}, + }, + Dependencies: []code.CompiledProto{ + {Name: "base", Node: &code.Node{ID: baseID}, Proto: compileTestProto(t, `return { v = 1 }`)}, + {Name: "mid", Node: &code.Node{ID: midID}, Proto: compileTestProto(t, `return { v = b.v + 1 }`)}, + }, + } +} + +// Repeated executions of one process neither add initializers, exported +// functions or registry entries nor grow the heap. +func TestPooledExecutionsDoNotGrowState(t *testing.T) { + proc := newDependencyProcess(t, libraryCompiled(t), `return { main = function() return m.v end }`, 50) + proc.EnablePreemption() + + run := func(n int) { + for i := 0; i < n; i++ { + if err := proc.Init(frameContext(), "main", nil); err != nil { + t.Fatal(err) + } + var output process.StepOutput + for j := 0; ; j++ { + output.Reset() + if err := proc.Step(nil, &output); err != nil { + t.Fatal(err) + } + if output.Status() == process.StepDone { + break + } + if j > 100 { + t.Fatal("execution did not finish") + } + } + } + } + + run(50) + list := initializersOf(proc.State()) + items, complete := len(list.items), len(list.onComplete) + exported := len(proc.exported) + top := proc.State().GetTop() + heap0 := heapInUse() + + for batch := 0; batch < 4; batch++ { + run(2000) + } + + list = initializersOf(proc.State()) + if len(list.items) != items || len(list.onComplete) != complete { + t.Fatalf("initializer list grew: items %d->%d, onComplete %d->%d", items, len(list.items), complete, len(list.onComplete)) + } + if len(proc.exported) != exported { + t.Fatalf("exported cache grew: %d->%d", exported, len(proc.exported)) + } + if got := proc.State().GetTop(); got != top { + t.Fatalf("main stack grew: %d->%d", top, got) + } + if len(proc.threads) != 0 { + t.Fatalf("%d threads retained", len(proc.threads)) + } + heap1 := heapInUse() + if heap1 > heap0+heap0/2+1<<20 { + t.Fatalf("heap grew from %d to %d over 8000 executions", heap0, heap1) + } +} + +type finalizerProbe struct{ collected *atomic.Int64 } + +// Process.Close drops every reference this package adds, so a closed process +// keeps nothing of its Lua state alive and its fields are reset for reuse. +func TestCloseReleasesPreemptedProcessState(t *testing.T) { + var collected atomic.Int64 + const total = 200 + baseline := runtime.NumGoroutine() + + for i := 0; i < total; i++ { + probe := &finalizerProbe{collected: &collected} + runtime.SetFinalizer(probe, func(p *finalizerProbe) { p.collected.Add(1) }) + binder := wrapBinder(func(l *lua.LState) { + ud := l.NewUserData() + ud.Value = probe + l.SetGlobal("probe", ud) + }) + entry := luaapi.ExecutionBudgets{TickBudget: 20, TickBudgetSet: true, MaxSteps: 1000, MaxStepsSet: true} + proc := mustNewProcess(t, + WithModuleBinder(binder), + WithProcessExecutionBudgets(entry), + WithScript(` + local t = {} + local co = coroutine.wrap(function() while true do t[#t+1] = probe coroutine.yield() end end) + return { main = function() for i = 1, 1e9 do co() end end } + `, "spin.lua")) + proc.EnablePreemption() + if err := proc.Init(frameContext(), "main", nil); err != nil { + t.Fatal(err) + } + var output process.StepOutput + for j := 0; j < 5; j++ { + output.Reset() + if err := proc.Step(nil, &output); err != nil { + t.Fatal(err) + } + if output.Status() != process.StepPreempted { + t.Fatalf("expected preemption, got %v", output.Status()) + } + } + proc.Close() + + if proc.preempted != nil || proc.exportFn != nil || proc.exported != nil || proc.steps != 0 || + proc.budgets != (luaapi.ExecutionBudgets{}) || proc.entryBudgets != (luaapi.ExecutionBudgets{}) || proc.preemptive { + t.Fatalf("closed process retains state: %+v", proc) + } + v := reflect.ValueOf(proc).Elem() + for _, name := range []string{"mainTask", "state", "factory", "upgradeRequest", "pendingOutdated", "result", "execErr"} { + if f := v.FieldByName(name); f.IsValid() && !f.IsNil() { + t.Fatalf("closed process retains %s", name) + } + } + } + + deadline := time.Now().Add(10 * time.Second) + for collected.Load() < total && time.Now().Before(deadline) { + runtime.GC() + time.Sleep(10 * time.Millisecond) + } + if got := collected.Load(); got != total { + t.Fatalf("%d of %d closed processes were collected", got, total) + } + if got := settleGoroutines(baseline); got > baseline { + t.Fatalf("goroutines grew from %d to %d", baseline, got) + } +} + +// Closing a process with drained initializers, a preempted task and queued +// messages across many cycles keeps the heap flat. +func TestSpawnPreemptCloseCyclesKeepHeapFlat(t *testing.T) { + compiled := libraryCompiled(t) + cycle := func(n int) { + for i := 0; i < n; i++ { + binder := NewProcessFactory(nil).isolationBinder(compiled, newProcessConfig(), nil, nil, nil, nil) + proc := mustNewProcess(t, + WithModuleBinder(wrapBinder(func(l *lua.LState) { lua.OpenErrors(l) })), + WithModuleBinder(binder), + WithProcessExecutionBudgets(luaapi.ExecutionBudgets{TickBudget: 10, TickBudgetSet: true}), + WithScript(fmt.Sprintf(`return { main = function() local n = %d while true do n = n + m.v end end }`, i), "m.lua")) + proc.EnablePreemption() + if err := proc.Init(frameContext(), "main", nil); err != nil { + t.Fatal(err) + } + var output process.StepOutput + for j := 0; j < 6; j++ { + output.Reset() + if err := proc.Step(nil, &output); err != nil { + t.Fatal(err) + } + } + proc.Close() + } + } + cycle(200) + heap0 := heapInUse() + for batch := 0; batch < 4; batch++ { + cycle(500) + } + heap1 := heapInUse() + if heap1 > heap0+heap0/2+1<<20 { + t.Fatalf("heap grew from %d to %d over 2000 cycles", heap0, heap1) + } +} + +type closeCounting struct { + *Process + closed *atomic.Int64 +} + +func (c *closeCounting) Close() { + c.closed.Add(1) + c.Process.Close() +} + +func (c *closeCounting) Send(*relay.Package) error { return nil } + +// Actors that finish, exceed max_steps or are terminated while preempting are +// all closed, unregistered and leave no goroutines behind. +func TestSchedulerLuaActorExitsReleaseEverything(t *testing.T) { + baseline := runtime.NumGoroutine() + var closed atomic.Int64 + executor := newBenchExecutorWithOptions(newMockRegistry(), scheduler.WithWorkers(4)) + executor.Start() + + finish := compileTestProto(t, `local s = 0 for i = 1, 3000 do s = s + i end return s`) + spin := compileTestProto(t, `while true do end`) + budgets := luaapi.ExecutionBudgets{TickBudget: 50, TickBudgetSet: true} + + const total = 300 + var wg sync.WaitGroup + results := make([]error, total) + for i := 0; i < total; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + proto := finish + if i%3 != 0 { + proto = spin + } + proc := mustNewProcess(t, WithProto(proto), WithProcessExecutionBudgets(budgets)) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + frame, fc := ctxapi.OpenFrameContext(ctx) + switch i % 3 { + case 1: + _ = fc.Set(apiruntime.FrameLifecycleOptionsKey, attrs.Attributes(attrs.Bag{"max_steps": int64(10)})) + case 2: + time.AfterFunc(20*time.Millisecond, cancel) + } + result, err := executor.Execute(frame, newTestPID(fmt.Sprintf("a%d", i)), &closeCounting{Process: proc, closed: &closed}, "", nil) + if err != nil { + results[i] = err + return + } + results[i] = result.Error + }(i) + } + wg.Wait() + + for i, err := range results { + switch i % 3 { + case 0: + if err != nil { + t.Fatalf("actor %d: %v", i, err) + } + case 1: + if !errors.Is(err, process.ErrStepLimitExceeded) { + t.Fatalf("actor %d: expected step limit, got %v", i, err) + } + case 2: + if err == nil { + t.Fatalf("actor %d: expected termination", i) + } + } + } + deadline := time.Now().Add(5 * time.Second) + for closed.Load() < total && time.Now().Before(deadline) { + time.Sleep(5 * time.Millisecond) + } + if got := closed.Load(); got != total { + t.Fatalf("closed %d of %d actors", got, total) + } + executor.Stop() + if got := settleGoroutines(baseline); got > baseline { + t.Fatalf("goroutines grew from %d to %d", baseline, got) + } +} diff --git a/system/scheduler/actor/preempt_leak_test.go b/system/scheduler/actor/preempt_leak_test.go new file mode 100644 index 000000000..687f02394 --- /dev/null +++ b/system/scheduler/actor/preempt_leak_test.go @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: MPL-2.0 + +package actor + +import ( + "context" + "fmt" + "runtime" + "sync/atomic" + "testing" + "time" + + "github.com/wippyai/runtime/api/payload" + pidapi "github.com/wippyai/runtime/api/pid" + "github.com/wippyai/runtime/api/process" + "github.com/wippyai/runtime/api/relay" + apiruntime "github.com/wippyai/runtime/api/runtime" +) + +// preemptNProcess reports StepPreempted n times, then completes. +type preemptNProcess struct { + closed *atomic.Int64 + left int +} + +func (*preemptNProcess) Init(context.Context, string, payload.Payloads) error { return nil } +func (p *preemptNProcess) Step(_ []process.Event, out *process.StepOutput) error { + if p.left == 0 { + out.Done(nil) + return nil + } + p.left-- + out.Preempt() + return nil +} +func (*preemptNProcess) Send(*relay.Package) error { return nil } +func (p *preemptNProcess) Close() { p.closed.Add(1) } + +func settledGoroutines(t *testing.T, want int) int { + t.Helper() + var n int + for i := 0; i < 200; i++ { + runtime.GC() + n = runtime.NumGoroutine() + if n <= want { + return n + } + time.Sleep(10 * time.Millisecond) + } + return n +} + +func TestPreemptCyclesReleaseEveryProcessAndGoroutine(t *testing.T) { + baseline := settledGoroutines(t, 0) + + var completed, closed atomic.Int64 + lc := &testLifecycle{ + onComplete: func(context.Context, pidapi.PID, *apiruntime.Result) { completed.Add(1) }, + } + sched := newPreemptTestScheduler(4, lc) + sched.Start() + + const total = 3000 + for i := 0; i < total; i++ { + p := &preemptNProcess{closed: &closed, left: i % 7} + id := pidapi.PID{UniqID: fmt.Sprintf("p%d", i)} + if _, err := sched.Submit(context.Background(), id, p, "", nil); err != nil { + t.Fatal(err) + } + } + + deadline := time.Now().Add(20 * time.Second) + for completed.Load() < total && time.Now().Before(deadline) { + time.Sleep(5 * time.Millisecond) + } + if got := completed.Load(); got != total { + t.Fatalf("completed %d of %d", got, total) + } + deadline = time.Now().Add(5 * time.Second) + for closed.Load() < total && time.Now().Before(deadline) { + time.Sleep(5 * time.Millisecond) + } + if got := closed.Load(); got != total { + t.Fatalf("closed %d of %d processes", got, total) + } + + // Idle workers are the only parked ones, each counted once. + deadline = time.Now().Add(5 * time.Second) + for sched.parked.Load() != 4 && time.Now().Before(deadline) { + time.Sleep(5 * time.Millisecond) + } + if got := sched.parked.Load(); got != 4 { + t.Fatalf("expected 4 parked workers while idle, got %d", got) + } + + testStopScheduler(sched) + if got := sched.parked.Load(); got != 0 { + t.Fatalf("parked counter drifted to %d after stop", got) + } + if got := settledGoroutines(t, baseline); got > baseline { + t.Fatalf("goroutines grew from %d to %d", baseline, got) + } +} + +// Processes stopped while still preempting are closed and unregistered. +func TestStopClosesPreemptingProcesses(t *testing.T) { + var closed atomic.Int64 + sched := newPreemptTestScheduler(2, &testLifecycle{}) + sched.Start() + const total = 20 + for i := 0; i < total; i++ { + p := &preemptNProcess{closed: &closed, left: 1 << 30} + if _, err := sched.Submit(context.Background(), pidapi.PID{UniqID: fmt.Sprintf("s%d", i)}, p, "", nil); err != nil { + t.Fatal(err) + } + } + time.Sleep(50 * time.Millisecond) + testStopScheduler(sched) + if got := closed.Load(); got != total { + t.Fatalf("closed %d of %d processes at stop", got, total) + } + if got := sched.parked.Load(); got != 0 { + t.Fatalf("parked counter is %d after stop", got) + } +} From 8bbc4e0ac370f3ba5afca47c32cdf227c12ee0a7 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Fri, 2 Oct 2026 22:55:45 -0400 Subject: [PATCH 19/50] style: order imports in close ownership test --- system/scheduler/actor/close_ownership_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/system/scheduler/actor/close_ownership_test.go b/system/scheduler/actor/close_ownership_test.go index 88c5b5aa7..b0971c463 100644 --- a/system/scheduler/actor/close_ownership_test.go +++ b/system/scheduler/actor/close_ownership_test.go @@ -5,19 +5,19 @@ package actor import ( "context" "errors" - "github.com/wippyai/runtime/api/dispatcher" - "github.com/wippyai/runtime/api/relay" - "github.com/wippyai/runtime/system/scheduler" "sync/atomic" "testing" "time" ctxapi "github.com/wippyai/runtime/api/context" + "github.com/wippyai/runtime/api/dispatcher" "github.com/wippyai/runtime/api/payload" pidapi "github.com/wippyai/runtime/api/pid" "github.com/wippyai/runtime/api/process" "github.com/wippyai/runtime/api/registry" + "github.com/wippyai/runtime/api/relay" apiruntime "github.com/wippyai/runtime/api/runtime" + "github.com/wippyai/runtime/system/scheduler" ) // closeCountingProcess counts Close calls. From 6d89b7caffe18673ed6929083bd5d5417b47c336 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Fri, 2 Oct 2026 23:38:24 -0400 Subject: [PATCH 20/50] chore(deps): go-lua with leak and pooling fixes --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 92e6aa1e4..1a6f740c3 100644 --- a/go.mod +++ b/go.mod @@ -69,7 +69,7 @@ require ( github.com/tree-sitter/tree-sitter-php v0.25.0 github.com/tree-sitter/tree-sitter-python v0.25.0 github.com/tree-sitter/tree-sitter-typescript v0.23.2 - github.com/wippyai/go-lua v1.6.4-0.20261003012915-c14848bbdb76 + github.com/wippyai/go-lua v1.6.4-0.20261003025238-2c373075d09d github.com/wippyai/module-registry-proto-go v0.0.2-0.20260908140534-f6e2910c835f github.com/wippyai/tree-sitter-markdown v0.0.3 github.com/wippyai/tree-sitter-sql v0.0.4 diff --git a/go.sum b/go.sum index 280da1f11..2e0f9f4e8 100644 --- a/go.sum +++ b/go.sum @@ -550,6 +550,8 @@ github.com/wippyai/go-lua v1.6.4-0.20261003005521-47a27bd56cfd h1:vmuy4HscyBNOQs github.com/wippyai/go-lua v1.6.4-0.20261003005521-47a27bd56cfd/go.mod h1:cwD+390gJgx9cs+Zamby/lch5csqPV9RN1aDEfx/27M= github.com/wippyai/go-lua v1.6.4-0.20261003012915-c14848bbdb76 h1:jTTXVVQ9q3HK2SUnFnzQj2SCInmKFTKTlE4XDdZ3WbQ= github.com/wippyai/go-lua v1.6.4-0.20261003012915-c14848bbdb76/go.mod h1:cwD+390gJgx9cs+Zamby/lch5csqPV9RN1aDEfx/27M= +github.com/wippyai/go-lua v1.6.4-0.20261003025238-2c373075d09d h1:NpoQ7a1P/B1u/hVLvV2l4IR9ESMsv0NWziLXmSzscIk= +github.com/wippyai/go-lua v1.6.4-0.20261003025238-2c373075d09d/go.mod h1:cwD+390gJgx9cs+Zamby/lch5csqPV9RN1aDEfx/27M= github.com/wippyai/module-registry-proto-go v0.0.2-0.20260908140534-f6e2910c835f h1:AoKS058roqnsHXaJ8Gww7ODz14qJqOK5QBeoALvcq6I= github.com/wippyai/module-registry-proto-go v0.0.2-0.20260908140534-f6e2910c835f/go.mod h1:p4ihYQKRQuqRVLxaH1YL0IEnkz4zKdyfVADOwvA5Tno= github.com/wippyai/tree-sitter-markdown v0.0.3 h1:u6e53+hzPSS848Xhm+I48SwtvWQHrC21wDrogdwCwuc= From d96d880da9f03676cedfb61c3f29266da70b2c15 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sat, 3 Oct 2026 00:34:59 -0400 Subject: [PATCH 21/50] fix(actor): publish a processor as Ready only once it is initialized Submit and CreateProcessor stored StateReady before setting the context, cancel function, queue generation and completer, and the processor pool built processors in the Ready state. A stale queue entry left from a pooled processor's previous incarnation could win Ready->Running and step the new process half-initialized, which the race detector reported as a write of the context racing the worker's read. Processors are built and pooled as Complete, and Ready is the last store before the processor is scheduled. --- system/scheduler/actor/processor.go | 2 + system/scheduler/actor/scheduler.go | 7 ++- .../actor/submit_publication_test.go | 59 +++++++++++++++++++ 3 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 system/scheduler/actor/submit_publication_test.go diff --git a/system/scheduler/actor/processor.go b/system/scheduler/actor/processor.go index f8e9e0056..09573194a 100644 --- a/system/scheduler/actor/processor.go +++ b/system/scheduler/actor/processor.go @@ -218,6 +218,8 @@ var processorPool = sync.Pool{ queue: process.NewEventQueue(), } p.lastWorker.Store(noWorkerAffinity) + // A processor becomes Ready only when its submitter publishes it. + p.state.Store(int32(StateComplete)) return p }, } diff --git a/system/scheduler/actor/scheduler.go b/system/scheduler/actor/scheduler.go index 1c24402a3..666c574b0 100644 --- a/system/scheduler/actor/scheduler.go +++ b/system/scheduler/actor/scheduler.go @@ -335,7 +335,6 @@ func (s *Scheduler) Submit(ctx context.Context, pid pid.PID, p process.Process, proc.id = s.nextID.Add(1) proc.pid = pid proc.Process = p - proc.state.Store(int32(StateReady)) proc.ctx = procCtx proc.cancel = cancel proc.scheduler = s @@ -377,6 +376,9 @@ func (s *Scheduler) Submit(ctx context.Context, pid pid.PID, p process.Process, }) proc.cancel = func() { stopWake(); cancel() } + // A pooled processor can still be referenced by a stale queue entry from + // its previous incarnation; Ready is published only once it is initialized. + proc.state.Store(int32(StateReady)) s.global.Push(proc) s.wakeAny() @@ -470,7 +472,6 @@ func (s *Scheduler) CreateProcessor(ctx context.Context, pid pid.PID, p process. proc.id = s.nextID.Add(1) proc.pid = pid proc.Process = p - proc.state.Store(int32(StateReady)) proc.ctx = procCtx proc.cancel = cancel proc.scheduler = s @@ -488,6 +489,8 @@ func (s *Scheduler) CreateProcessor(ctx context.Context, pid pid.PID, p process. s.processorCount.Add(1) s.byPID.Store(pid.String(), proc) s.byQueue.Store(proc.queue, proc) + // Ready is published only once the processor is initialized; see Submit. + proc.state.Store(int32(StateReady)) return proc, nil } diff --git a/system/scheduler/actor/submit_publication_test.go b/system/scheduler/actor/submit_publication_test.go new file mode 100644 index 000000000..1a20dd117 --- /dev/null +++ b/system/scheduler/actor/submit_publication_test.go @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: MPL-2.0 + +package actor + +import ( + "context" + "sync/atomic" + "testing" + + "github.com/wippyai/runtime/api/payload" + pidapi "github.com/wippyai/runtime/api/pid" + "github.com/wippyai/runtime/api/process" + "github.com/wippyai/runtime/api/relay" +) + +// stepRecorder counts the steps it runs. +type stepRecorder struct{ steps atomic.Int32 } + +func (*stepRecorder) Init(context.Context, string, payload.Payloads) error { return nil } + +func (p *stepRecorder) Step(_ []process.Event, out *process.StepOutput) error { + p.steps.Add(1) + out.Done(nil) + return nil +} + +func (*stepRecorder) Send(*relay.Package) error { return nil } +func (*stepRecorder) Close() {} + +// A pooled processor can still sit in a worker's queue from its previous +// incarnation. Until Submit has initialized it, popping that stale entry +// must not run the new process. +func TestSubmitPublishesProcessorOnlyWhenInitialized(t *testing.T) { + proc := &stepRecorder{} + self := pidapi.PID{UniqID: "half-initialized"} + var sched *Scheduler + var ranEarly int32 + sched = newTestSchedulerWithLifecycle(1, &testLifecycle{ + onStart: func(context.Context, pidapi.PID, process.Process) { + v, ok := sched.byPID.Load(self.String()) + if !ok { + t.Error("the processor is registered before lifecycle start") + return + } + // A worker pops the stale entry while Submit is still running. + sched.workerSnapshot()[0].executeOne(v.(*Processor)) + ranEarly = proc.steps.Load() + }, + }) + sched.Start() + defer testStopScheduler(sched) + + if _, err := sched.Submit(context.Background(), self, proc, "", nil); err != nil { + t.Fatal(err) + } + if ranEarly != 0 { + t.Fatalf("a stale queue entry ran the process before Submit initialized it (%d steps)", ranEarly) + } +} From a8d37c721191ca451c7934815866203b0cbc897b Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sat, 3 Oct 2026 09:50:48 -0400 Subject: [PATCH 22/50] chore(deps): go-lua with bounded call stack frames --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 1a6f740c3..bb33f19b4 100644 --- a/go.mod +++ b/go.mod @@ -69,7 +69,7 @@ require ( github.com/tree-sitter/tree-sitter-php v0.25.0 github.com/tree-sitter/tree-sitter-python v0.25.0 github.com/tree-sitter/tree-sitter-typescript v0.23.2 - github.com/wippyai/go-lua v1.6.4-0.20261003025238-2c373075d09d + github.com/wippyai/go-lua v1.6.4-0.20261003135019-e6c9f2e920e8 github.com/wippyai/module-registry-proto-go v0.0.2-0.20260908140534-f6e2910c835f github.com/wippyai/tree-sitter-markdown v0.0.3 github.com/wippyai/tree-sitter-sql v0.0.4 diff --git a/go.sum b/go.sum index 2e0f9f4e8..ec807ebe9 100644 --- a/go.sum +++ b/go.sum @@ -552,6 +552,8 @@ github.com/wippyai/go-lua v1.6.4-0.20261003012915-c14848bbdb76 h1:jTTXVVQ9q3HK2S github.com/wippyai/go-lua v1.6.4-0.20261003012915-c14848bbdb76/go.mod h1:cwD+390gJgx9cs+Zamby/lch5csqPV9RN1aDEfx/27M= github.com/wippyai/go-lua v1.6.4-0.20261003025238-2c373075d09d h1:NpoQ7a1P/B1u/hVLvV2l4IR9ESMsv0NWziLXmSzscIk= github.com/wippyai/go-lua v1.6.4-0.20261003025238-2c373075d09d/go.mod h1:cwD+390gJgx9cs+Zamby/lch5csqPV9RN1aDEfx/27M= +github.com/wippyai/go-lua v1.6.4-0.20261003135019-e6c9f2e920e8 h1:cPF3sBY4gbJi5wDrecxyjkdjtgn4qAFAtnuPsbyO/Kw= +github.com/wippyai/go-lua v1.6.4-0.20261003135019-e6c9f2e920e8/go.mod h1:cwD+390gJgx9cs+Zamby/lch5csqPV9RN1aDEfx/27M= github.com/wippyai/module-registry-proto-go v0.0.2-0.20260908140534-f6e2910c835f h1:AoKS058roqnsHXaJ8Gww7ODz14qJqOK5QBeoALvcq6I= github.com/wippyai/module-registry-proto-go v0.0.2-0.20260908140534-f6e2910c835f/go.mod h1:p4ihYQKRQuqRVLxaH1YL0IEnkz4zKdyfVADOwvA5Tno= github.com/wippyai/tree-sitter-markdown v0.0.3 h1:u6e53+hzPSS848Xhm+I48SwtvWQHrC21wDrogdwCwuc= From 82715d39d70dcaf98b2521769a4e501389ef6a08 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sat, 3 Oct 2026 10:01:18 -0400 Subject: [PATCH 23/50] test(actor): check stale completions without relying on pool reuse The test waited for sync.Pool to hand the finished processor to a new process, which the race detector's randomized pool drops make nondeterministic. It now asserts that yield handlers receive the incarnation's generation-bound completer; the queue test covers that a stale completer is inert. --- .../scheduler/actor/close_ownership_test.go | 54 ++++++------------- 1 file changed, 17 insertions(+), 37 deletions(-) diff --git a/system/scheduler/actor/close_ownership_test.go b/system/scheduler/actor/close_ownership_test.go index b0971c463..e1df23e0a 100644 --- a/system/scheduler/actor/close_ownership_test.go +++ b/system/scheduler/actor/close_ownership_test.go @@ -136,6 +136,10 @@ func (*waitingProcess) Close() {} // A completion that arrives after its actor ended is dropped, even when the // processor slot has been reused by another actor. +// Yield handlers complete through the incarnation's generation-bound +// completer, never the pooled Processor: a completion that outlives the +// incarnation is dropped by the queue (TestA03StaleCompletionDoesNotEnqueue) +// instead of reaching the processor's next user. func TestStaleYieldCompletionDoesNotWakeReusedProcessor(t *testing.T) { received := make(chan dispatcher.ResultReceiver, 1) reg := scheduler.NewRegistry() @@ -147,54 +151,30 @@ func TestStaleYieldCompletionDoesNotWakeReusedProcessor(t *testing.T) { return nil })) reg.Register(CmdComplete, CompleteHandler()) - done := make(chan struct{}, 64) + done := make(chan struct{}, 1) lc := &testLifecycle{onComplete: func(context.Context, pidapi.PID, *apiruntime.Result) { done <- struct{}{} }} sched := NewScheduler(reg, WithWorkers(1), WithLifecycle(lc)) sched.Start() defer testStopScheduler(sched) - ctxA, cancelA := context.WithCancel(context.Background()) - a := &waitingProcess{} - procA, err := sched.Submit(ctxA, pidapi.PID{UniqID: "a"}, a, "", nil) - if err != nil { + ctx, cancel := context.WithCancel(context.Background()) + if _, err := sched.Submit(ctx, pidapi.PID{UniqID: "a"}, &waitingProcess{}, "", nil); err != nil { t.Fatal(err) } - var stale dispatcher.ResultReceiver + var receiver dispatcher.ResultReceiver select { - case stale = <-received: + case receiver = <-received: case <-time.After(5 * time.Second): t.Fatal("yield was not dispatched") } - cancelA() + completer, ok := receiver.(*process.YieldCompleter) + if !ok { + t.Fatalf("handlers complete through the incarnation's completer, got %T", receiver) + } + cancel() <-done - for i := 0; i < 1000; i++ { - ctxB, cancelB := context.WithCancel(context.Background()) - b := &waitingProcess{} - procB, err := sched.Submit(ctxB, pidapi.PID{UniqID: "b"}, b, "", nil) - if err != nil { - t.Fatal(err) - } - for b.steps.Load() == 0 { - time.Sleep(time.Millisecond) - } - select { - case <-received: - case <-time.After(5 * time.Second): - t.Fatal("yield was not dispatched") - } - if procB == procA { - stale.CompleteYield(7, nil, nil) - time.Sleep(50 * time.Millisecond) - if got := b.steps.Load(); got != 1 { - t.Fatalf("stale completion woke the reused processor: %d steps", got) - } - cancelB() - <-done - return - } - cancelB() - <-done - } - t.Fatal("processor slot was never reused") + // After completion the processor may serve another process; the old + // completer stays bound to the finished incarnation. + completer.CompleteYield(7, nil, nil) } From 59f617b444a744c577cb016f9c9f1faa45b395b2 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sat, 3 Oct 2026 10:30:49 -0400 Subject: [PATCH 24/50] fix(lua): count steps for processes without a step limit StepsUsed reported zero for actors without max_steps, so an upgrade into a definition that sets a limit restarted the lifetime count. --- runtime/lua/engine/preempt_test.go | 23 +++++++++++++++++++++++ runtime/lua/engine/process.go | 8 +++----- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/runtime/lua/engine/preempt_test.go b/runtime/lua/engine/preempt_test.go index a9c58f18b..02163afcd 100644 --- a/runtime/lua/engine/preempt_test.go +++ b/runtime/lua/engine/preempt_test.go @@ -533,6 +533,29 @@ func TestProcessStepCountContinuesAcrossResume(t *testing.T) { } } +func TestProcessStepsUsedCountsStepsWithoutLimit(t *testing.T) { + script := ` + local s = 0 + for i = 1, 100000 do s = s + i end + return s + ` + entry := luaapi.ExecutionBudgets{TickBudget: 100, TickBudgetSet: true} + proc, err := initWithSpawnOptions(t, script, entry, nil) + if err != nil { + t.Fatal(err) + } + var output process.StepOutput + for i := 0; i < 3; i++ { + output.Reset() + if err := proc.Step(nil, &output); err != nil { + t.Fatal(err) + } + } + if got := proc.StepsUsed(); got != 3 { + t.Fatalf("expected 3 steps used, got %d", got) + } +} + type upgradeFactory struct { create func() (process.Process, *process.Meta, error) } diff --git a/runtime/lua/engine/process.go b/runtime/lua/engine/process.go index a5b8b3692..c20cc675a 100644 --- a/runtime/lua/engine/process.go +++ b/runtime/lua/engine/process.go @@ -976,13 +976,11 @@ func (p *Process) stepTickBudget() int64 { } } -// chargeStep counts a step against the max_steps option. +// chargeStep counts a step and fails it when the count exceeds a non-zero +// max_steps option. func (p *Process) chargeStep() error { - if p.budgets.MaxSteps == 0 { - return nil - } p.steps++ - if p.steps > p.budgets.MaxSteps { + if p.budgets.MaxSteps != 0 && p.steps > p.budgets.MaxSteps { return process.ErrStepLimitExceeded } return nil From 52f0ba59bc350b633e96cf28cba6fc79a35c7128 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sat, 3 Oct 2026 10:30:59 -0400 Subject: [PATCH 25/50] refactor(lua,actor): share initializer list creation, make enablePreemption a function --- runtime/lua/engine/initializer.go | 21 +++++++++++---------- system/scheduler/actor/scheduler.go | 6 +++--- system/scheduler/actor/worker.go | 2 +- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/runtime/lua/engine/initializer.go b/runtime/lua/engine/initializer.go index 93a1a0a45..87f3514ec 100644 --- a/runtime/lua/engine/initializer.go +++ b/runtime/lua/engine/initializer.go @@ -31,11 +31,7 @@ type Initializer struct { // Call it from a ModuleBinder. Registration order must follow dependency // order, so a library's imports are initialized before the library. func DeferInitializer(l *lua.LState, init Initializer) { - list := initializersOf(l) - if list == nil { - list = &initializerList{} - l.G.Registry.RawSetString(initializersRegistryKey, list) - } + list := ensureInitializers(l) list.items = append(list.items, init) } @@ -58,11 +54,7 @@ func (*initializerList) Type() lua.LValueType { return lua.LTUserData } // completed, before the entry chunk runs. It does not run if none are // registered with DeferInitializer. func OnInitialized(l *lua.LState, fn func()) { - list := initializersOf(l) - if list == nil { - list = &initializerList{} - l.G.Registry.RawSetString(initializersRegistryKey, list) - } + list := ensureInitializers(l) list.onComplete = append(list.onComplete, fn) } @@ -71,6 +63,15 @@ func initializersOf(l *lua.LState) *initializerList { return list } +func ensureInitializers(l *lua.LState) *initializerList { + list := initializersOf(l) + if list == nil { + list = &initializerList{} + l.G.Registry.RawSetString(initializersRegistryKey, list) + } + return list +} + func (list *initializerList) pending() bool { return list != nil && list.done < len(list.items) } diff --git a/system/scheduler/actor/scheduler.go b/system/scheduler/actor/scheduler.go index 666c574b0..c3c48f90e 100644 --- a/system/scheduler/actor/scheduler.go +++ b/system/scheduler/actor/scheduler.go @@ -240,7 +240,7 @@ func (s *Scheduler) wakeAny() { // enablePreemption lets a preemptible process suspend long steps: the // scheduler runs it again after StepPreempted. -func (s *Scheduler) enablePreemption(p process.Process) { +func enablePreemption(p process.Process) { if pp, ok := p.(process.Preemptible); ok { pp.EnablePreemption() } @@ -325,7 +325,7 @@ func (s *Scheduler) Submit(ctx context.Context, pid pid.PID, p process.Process, procCtx, cancel = context.WithCancel(ctx) } - s.enablePreemption(p) + enablePreemption(p) if err := p.Init(procCtx, method, input); err != nil { cancel() return nil, err @@ -463,7 +463,7 @@ func (s *Scheduler) CreateProcessor(ctx context.Context, pid pid.PID, p process. return nil, process.ErrMaxProcessesExceeded } - s.enablePreemption(p) + enablePreemption(p) // Wrap context with cancel for Terminate support procCtx, cancel := context.WithCancel(ctx) diff --git a/system/scheduler/actor/worker.go b/system/scheduler/actor/worker.go index 1b2060deb..069de952b 100644 --- a/system/scheduler/actor/worker.go +++ b/system/scheduler/actor/worker.go @@ -529,7 +529,7 @@ func (w *Worker) executeOne(proc *Processor) { // Swap proc.Process = newProc - w.scheduler.enablePreemption(newProc) + enablePreemption(newProc) // Init new process method := "main" From a23214121d0aa055afe11b9865c12ca5df029e8b Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sat, 3 Oct 2026 10:33:15 -0400 Subject: [PATCH 26/50] test(actor): wait on completion signals instead of polling and sleeping Document the Set flags of ExecutionBudgets and drop a fairness comment that contradicts guest preemption. --- api/runtime/lua/options.go | 6 +- .../scheduler/actor/close_ownership_test.go | 3 +- system/scheduler/actor/preempt_leak_test.go | 51 +++++---- system/scheduler/actor/preempt_test.go | 102 +++++++++--------- system/scheduler/actor/worker.go | 2 +- 5 files changed, 86 insertions(+), 78 deletions(-) diff --git a/api/runtime/lua/options.go b/api/runtime/lua/options.go index 437b78125..0ea1f588b 100644 --- a/api/runtime/lua/options.go +++ b/api/runtime/lua/options.go @@ -26,8 +26,10 @@ const ( // MaxSteps bounds the number of scheduler steps the actor may take; zero is // unlimited and exceeding it fails the actor with process.ErrStepLimitExceeded. type ExecutionBudgets struct { - TickBudget int64 - MaxSteps uint64 + TickBudget int64 + MaxSteps uint64 + // TickBudgetSet and MaxStepsSet report that the option was given, so an + // explicit zero overrides a lower-precedence value. TickBudgetSet bool MaxStepsSet bool } diff --git a/system/scheduler/actor/close_ownership_test.go b/system/scheduler/actor/close_ownership_test.go index e1df23e0a..d83779935 100644 --- a/system/scheduler/actor/close_ownership_test.go +++ b/system/scheduler/actor/close_ownership_test.go @@ -82,7 +82,8 @@ func TestFailedUpgradeClosesProcessOnce(t *testing.T) { case <-time.After(5 * time.Second): t.Fatal("actor did not complete") } - time.Sleep(50 * time.Millisecond) + // Stopping joins the workers, so every Close has happened. + testStopScheduler(sched) closes := source.closed.Load() if target != nil { closes += target.closed.Load() diff --git a/system/scheduler/actor/preempt_leak_test.go b/system/scheduler/actor/preempt_leak_test.go index 687f02394..4eb87bcb9 100644 --- a/system/scheduler/actor/preempt_leak_test.go +++ b/system/scheduler/actor/preempt_leak_test.go @@ -20,11 +20,20 @@ import ( // preemptNProcess reports StepPreempted n times, then completes. type preemptNProcess struct { closed *atomic.Int64 - left int + // started, when set, receives a value on the first step. + started chan<- struct{} + left int + began bool } func (*preemptNProcess) Init(context.Context, string, payload.Payloads) error { return nil } func (p *preemptNProcess) Step(_ []process.Event, out *process.StepOutput) error { + if !p.began { + p.began = true + if p.started != nil { + p.started <- struct{}{} + } + } if p.left == 0 { out.Done(nil) return nil @@ -53,14 +62,15 @@ func settledGoroutines(t *testing.T, want int) int { func TestPreemptCyclesReleaseEveryProcessAndGoroutine(t *testing.T) { baseline := settledGoroutines(t, 0) - var completed, closed atomic.Int64 + const total = 3000 + var closed atomic.Int64 + completed := make(chan struct{}, total) lc := &testLifecycle{ - onComplete: func(context.Context, pidapi.PID, *apiruntime.Result) { completed.Add(1) }, + onComplete: func(context.Context, pidapi.PID, *apiruntime.Result) { completed <- struct{}{} }, } sched := newPreemptTestScheduler(4, lc) sched.Start() - const total = 3000 for i := 0; i < total; i++ { p := &preemptNProcess{closed: &closed, left: i % 7} id := pidapi.PID{UniqID: fmt.Sprintf("p%d", i)} @@ -69,29 +79,27 @@ func TestPreemptCyclesReleaseEveryProcessAndGoroutine(t *testing.T) { } } - deadline := time.Now().Add(20 * time.Second) - for completed.Load() < total && time.Now().Before(deadline) { - time.Sleep(5 * time.Millisecond) - } - if got := completed.Load(); got != total { - t.Fatalf("completed %d of %d", got, total) - } - deadline = time.Now().Add(5 * time.Second) - for closed.Load() < total && time.Now().Before(deadline) { - time.Sleep(5 * time.Millisecond) - } - if got := closed.Load(); got != total { - t.Fatalf("closed %d of %d processes", got, total) + timeout := time.After(20 * time.Second) + for i := 0; i < total; i++ { + select { + case <-completed: + case <-timeout: + t.Fatalf("completed %d of %d", i, total) + } } // Idle workers are the only parked ones, each counted once. - deadline = time.Now().Add(5 * time.Second) + deadline := time.Now().Add(5 * time.Second) for sched.parked.Load() != 4 && time.Now().Before(deadline) { time.Sleep(5 * time.Millisecond) } if got := sched.parked.Load(); got != 4 { t.Fatalf("expected 4 parked workers while idle, got %d", got) } + // A parked worker has returned from its last step, so every Close ran. + if got := closed.Load(); got != total { + t.Fatalf("closed %d of %d processes", got, total) + } testStopScheduler(sched) if got := sched.parked.Load(); got != 0 { @@ -108,13 +116,16 @@ func TestStopClosesPreemptingProcesses(t *testing.T) { sched := newPreemptTestScheduler(2, &testLifecycle{}) sched.Start() const total = 20 + started := make(chan struct{}, total) for i := 0; i < total; i++ { - p := &preemptNProcess{closed: &closed, left: 1 << 30} + p := &preemptNProcess{closed: &closed, left: 1 << 30, started: started} if _, err := sched.Submit(context.Background(), pidapi.PID{UniqID: fmt.Sprintf("s%d", i)}, p, "", nil); err != nil { t.Fatal(err) } } - time.Sleep(50 * time.Millisecond) + for i := 0; i < total; i++ { + <-started + } testStopScheduler(sched) if got := closed.Load(); got != total { t.Fatalf("closed %d of %d processes at stop", got, total) diff --git a/system/scheduler/actor/preempt_test.go b/system/scheduler/actor/preempt_test.go index 5a55af72b..fa5c18f18 100644 --- a/system/scheduler/actor/preempt_test.go +++ b/system/scheduler/actor/preempt_test.go @@ -83,6 +83,19 @@ func (p *preemptWithPendingYieldProcess) Step(_ []process.Event, out *process.St func (*preemptWithPendingYieldProcess) Send(*relay.Package) error { return nil } func (*preemptWithPendingYieldProcess) Close() {} +// newPingLifecycle returns a lifecycle whose channel closes when the process +// with pid "ping" completes. +func newPingLifecycle() (*testLifecycle, <-chan struct{}) { + done := make(chan struct{}) + return &testLifecycle{ + onComplete: func(_ context.Context, pid pidapi.PID, _ *runtime.Result) { + if pid.UniqID == "ping" { + close(done) + } + }, + }, done +} + func newPreemptTestScheduler(workers int, lc *testLifecycle, opts ...Option) *Scheduler { registry := scheduler.NewRegistry() registry.Register(CmdComplete, CompleteHandler()) @@ -116,14 +129,7 @@ func TestSchedulerEnablesPreemptionForPreemptibleProcesses(t *testing.T) { // A preempted process goes behind other ready work: on a single worker a // process that keeps yielding completes while a spinning process runs. func TestPreemptedProcessYieldsWorkerToReadyProcesses(t *testing.T) { - var pingDone atomic.Bool - lc := &testLifecycle{ - onComplete: func(_ context.Context, pid pidapi.PID, _ *runtime.Result) { - if pid.UniqID == "ping" { - pingDone.Store(true) - } - }, - } + lc, pingDone := newPingLifecycle() sched := newPreemptTestScheduler(1, lc) sched.Start() defer testStopScheduler(sched) @@ -138,11 +144,9 @@ func TestPreemptedProcessYieldsWorkerToReadyProcesses(t *testing.T) { t.Fatal(err) } - deadline := time.Now().Add(5 * time.Second) - for !pingDone.Load() && time.Now().Before(deadline) { - time.Sleep(time.Millisecond) - } - if !pingDone.Load() { + select { + case <-pingDone: + case <-time.After(5 * time.Second): t.Fatalf("ready process starved by preempted process (%d spin steps)", spin.steps.Load()) } } @@ -179,9 +183,9 @@ func TestPreemptedProcessIsQueuedBehindLocalReadyWork(t *testing.T) { } func TestPreemptedProcessRunsAgainWithPendingYields(t *testing.T) { - var done atomic.Bool + done := make(chan struct{}) lc := &testLifecycle{ - onComplete: func(context.Context, pidapi.PID, *runtime.Result) { done.Store(true) }, + onComplete: func(context.Context, pidapi.PID, *runtime.Result) { close(done) }, } sched := newPreemptTestScheduler(1, lc) sched.Start() @@ -191,11 +195,9 @@ func TestPreemptedProcessRunsAgainWithPendingYields(t *testing.T) { if _, err := sched.Submit(context.Background(), testPID(), p, "", nil); err != nil { t.Fatal(err) } - deadline := time.Now().Add(2 * time.Second) - for !done.Load() && time.Now().Before(deadline) { - time.Sleep(time.Millisecond) - } - if !done.Load() { + select { + case <-done: + case <-time.After(5 * time.Second): t.Fatalf("preempted process with a pending yield was not run again (%d steps)", p.steps.Load()) } } @@ -212,16 +214,21 @@ func (p *upgradeSource) ResumeStepCount(n uint64) { p.used = n } // upgradeTarget records how the scheduler prepared it as an upgrade replacement. type upgradeTarget struct { - stop chan struct{} - options atomic.Value - resumed atomic.Uint64 - enabled atomic.Bool - initialized atomic.Bool - spinning bool + stop chan struct{} + options atomic.Value + // counted closes when the scheduler calls ResumeStepCount, the last + // preparation step of an upgrade replacement. + counted chan struct{} + resumed atomic.Uint64 + enabled atomic.Bool + spinning bool +} + +func newUpgradeTarget() *upgradeTarget { + return &upgradeTarget{counted: make(chan struct{})} } func (p *upgradeTarget) Init(ctx context.Context, _ string, _ payload.Payloads) error { - p.initialized.Store(true) if opts := runtime.GetFrameLifecycleOptions(ctx); opts != nil { p.options.Store(opts) } @@ -231,6 +238,7 @@ func (p *upgradeTarget) EnablePreemption() { p.enabled.Store(true) } func (p *upgradeTarget) StepsUsed() uint64 { return p.resumed.Load() } func (p *upgradeTarget) ResumeStepCount(used uint64) { p.resumed.Store(used) + close(p.counted) } func (p *upgradeTarget) Step(_ []process.Event, out *process.StepOutput) error { if p.spinning { @@ -251,7 +259,7 @@ func (*upgradeTarget) Send(*relay.Package) error { return nil } func (*upgradeTarget) Close() {} // upgradeInto submits an upgrade source whose replacement is target and waits -// until the replacement is initialized. +// until the scheduler has initialized it and carried the step count over. func upgradeInto(ctx context.Context, t *testing.T, sched *Scheduler, source process.Process, target *upgradeTarget) { t.Helper() appCtx := ctxapi.WithAppContext(ctx, ctxapi.NewAppContext()) @@ -265,11 +273,9 @@ func upgradeInto(ctx context.Context, t *testing.T, sched *Scheduler, source pro if _, err := sched.Submit(frameCtx, pidapi.PID{UniqID: "upgrader"}, source, "", nil); err != nil { t.Fatal(err) } - deadline := time.Now().Add(5 * time.Second) - for !target.initialized.Load() && time.Now().Before(deadline) { - time.Sleep(time.Millisecond) - } - if !target.initialized.Load() { + select { + case <-target.counted: + case <-time.After(5 * time.Second): t.Fatal("upgrade replacement was not initialized") } } @@ -279,7 +285,7 @@ func TestUpgradeReplacementIsPreemptionEnabled(t *testing.T) { sched.Start() defer testStopScheduler(sched) - target := &upgradeTarget{} + target := newUpgradeTarget() source := &upgradeSource{UpgradeProcess: UpgradeProcess{upgradeReq: &process.UpgradeRequest{Source: registry.NewID("app", "next")}}} upgradeInto(context.Background(), t, sched, source, target) if !target.enabled.Load() { @@ -289,19 +295,13 @@ func TestUpgradeReplacementIsPreemptionEnabled(t *testing.T) { // An actor that upgrades into a spinning definition still yields its worker. func TestUpgradeIntoSpinningActorYieldsWorker(t *testing.T) { - var pingDone atomic.Bool - lc := &testLifecycle{ - onComplete: func(_ context.Context, pid pidapi.PID, _ *runtime.Result) { - if pid.UniqID == "ping" { - pingDone.Store(true) - } - }, - } + lc, pingDone := newPingLifecycle() sched := newPreemptTestScheduler(1, lc) sched.Start() defer testStopScheduler(sched) - target := &upgradeTarget{spinning: true, stop: make(chan struct{})} + target := newUpgradeTarget() + target.spinning, target.stop = true, make(chan struct{}) defer close(target.stop) source := &upgradeSource{UpgradeProcess: UpgradeProcess{upgradeReq: &process.UpgradeRequest{Source: registry.NewID("app", "next")}}} upgradeInto(context.Background(), t, sched, source, target) @@ -309,11 +309,9 @@ func TestUpgradeIntoSpinningActorYieldsWorker(t *testing.T) { if _, err := sched.Submit(context.Background(), pidapi.PID{UniqID: "ping"}, &pingProcess{rounds: 20}, "", nil); err != nil { t.Fatal(err) } - deadline := time.Now().Add(5 * time.Second) - for !pingDone.Load() && time.Now().Before(deadline) { - time.Sleep(time.Millisecond) - } - if !pingDone.Load() { + select { + case <-pingDone: + case <-time.After(5 * time.Second): t.Fatal("ready process starved by the upgraded actor") } } @@ -328,7 +326,7 @@ func TestUpgradeKeepsSpawnLifecycleOptions(t *testing.T) { if err := fc.Set(runtime.FrameLifecycleOptionsKey, attrs.Attributes(spawn)); err != nil { t.Fatal(err) } - target := &upgradeTarget{} + target := newUpgradeTarget() source := &upgradeSource{UpgradeProcess: UpgradeProcess{upgradeReq: &process.UpgradeRequest{Source: registry.NewID("app", "next")}}} upgradeInto(ctx, t, sched, source, target) @@ -343,16 +341,12 @@ func TestUpgradeCarriesStepCount(t *testing.T) { sched.Start() defer testStopScheduler(sched) - target := &upgradeTarget{} + target := newUpgradeTarget() source := &upgradeSource{ UpgradeProcess: UpgradeProcess{upgradeReq: &process.UpgradeRequest{Source: registry.NewID("app", "next")}}, used: 41, } upgradeInto(context.Background(), t, sched, source, target) - deadline := time.Now().Add(5 * time.Second) - for target.resumed.Load() == 0 && time.Now().Before(deadline) { - time.Sleep(time.Millisecond) - } if got := target.resumed.Load(); got != 41 { t.Fatalf("expected 41 steps carried to the replacement, got %d", got) } diff --git a/system/scheduler/actor/worker.go b/system/scheduler/actor/worker.go index 069de952b..68c6d8f5b 100644 --- a/system/scheduler/actor/worker.go +++ b/system/scheduler/actor/worker.go @@ -246,7 +246,7 @@ func (w *Worker) signal() bool { // localDispatchQuantum bounds preference for the cache-hot local continuation. // Fair turns rotate through injected wakeups, global submissions and the oldest -// local continuation. This is cooperative dispatch fairness, not guest preemption. +// local continuation. const localDispatchQuantum = 32 func (w *Worker) takeFairWork() *Processor { From de32f7d219954d6a5d32d45601f3e1b11fff2f5c Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sat, 3 Oct 2026 10:33:31 -0400 Subject: [PATCH 27/50] test(lua): assert actor closes after the scheduler stops --- runtime/lua/engine/preempt_leak_test.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/runtime/lua/engine/preempt_leak_test.go b/runtime/lua/engine/preempt_leak_test.go index 1b7faa80e..a62d3132d 100644 --- a/runtime/lua/engine/preempt_leak_test.go +++ b/runtime/lua/engine/preempt_leak_test.go @@ -295,14 +295,11 @@ func TestSchedulerLuaActorExitsReleaseEverything(t *testing.T) { } } } - deadline := time.Now().Add(5 * time.Second) - for closed.Load() < total && time.Now().Before(deadline) { - time.Sleep(5 * time.Millisecond) - } + // Stopping joins the workers, so every Close has happened. + executor.Stop() if got := closed.Load(); got != total { t.Fatalf("closed %d of %d actors", got, total) } - executor.Stop() if got := settleGoroutines(baseline); got > baseline { t.Fatalf("goroutines grew from %d to %d", baseline, got) } From da5068b4909bf9acad6b04b8ffa011fc5a9d58ef Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sat, 3 Oct 2026 10:33:40 -0400 Subject: [PATCH 28/50] refactor(api): read process entry options with Bag.GetBag The registry helper wrapped a nil-safe lookup that attrs.Bag already provides; api/runtime/lua no longer imports api/registry. --- api/registry/options.go | 17 ----------------- api/runtime/lua/options.go | 8 +++++--- 2 files changed, 5 insertions(+), 20 deletions(-) delete mode 100644 api/registry/options.go diff --git a/api/registry/options.go b/api/registry/options.go deleted file mode 100644 index 0e1aeb5dd..000000000 --- a/api/registry/options.go +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 - -package registry - -import "github.com/wippyai/runtime/api/attrs" - -// EntryOptionsMetaKey is the registry metadata bag holding declaration-time -// runtime options. Entry data remains the executable/config payload. -const EntryOptionsMetaKey = "options" - -// EntryOptionsFromMeta returns the nested meta.options bag. -func EntryOptionsFromMeta(meta attrs.Bag) (attrs.Bag, bool) { - if meta == nil { - return nil, false - } - return meta.GetBag(EntryOptionsMetaKey) -} diff --git a/api/runtime/lua/options.go b/api/runtime/lua/options.go index 0ea1f588b..dae6f294e 100644 --- a/api/runtime/lua/options.go +++ b/api/runtime/lua/options.go @@ -8,7 +8,6 @@ import ( "math" "github.com/wippyai/runtime/api/attrs" - "github.com/wippyai/runtime/api/registry" ) // Actor execution option keys, shared by process entry meta.options and @@ -18,6 +17,9 @@ const ( ProcessOptionMaxSteps = "max_steps" ) +// entryOptionsMetaKey is the entry meta bag holding declaration-time options. +const entryOptionsMetaKey = "options" + // ExecutionBudgets holds the execution limits of a Lua actor. // // TickBudget is the number of VM ticks (backward jumps, loop iterations and @@ -71,10 +73,10 @@ func ExecutionBudgetsFromOptions(options attrs.Attributes, subject string) (Exec func EntryExecutionBudgets(meta attrs.Bag) (ExecutionBudgets, error) { for _, key := range [...]string{ProcessOptionTickBudget, ProcessOptionMaxSteps} { if _, ok := meta.Get(key); ok { - return ExecutionBudgets{}, fmt.Errorf("process entry meta %q must be nested under meta.%s", key, registry.EntryOptionsMetaKey) + return ExecutionBudgets{}, fmt.Errorf("process entry meta %q must be nested under meta.%s", key, entryOptionsMetaKey) } } - options, ok := registry.EntryOptionsFromMeta(meta) + options, ok := meta.GetBag(entryOptionsMetaKey) if !ok { return ExecutionBudgets{}, nil } From 568fb0e1ee3df3b2a44cf5405ccb827c2b2b0654 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sat, 3 Oct 2026 10:33:57 -0400 Subject: [PATCH 29/50] test(lua): cover tick budget boundaries and exhaustion at task end --- runtime/lua/engine/preempt_test.go | 43 ++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/runtime/lua/engine/preempt_test.go b/runtime/lua/engine/preempt_test.go index 02163afcd..468c98be0 100644 --- a/runtime/lua/engine/preempt_test.go +++ b/runtime/lua/engine/preempt_test.go @@ -6,6 +6,7 @@ import ( "context" "errors" "fmt" + "math" "sync/atomic" "testing" "time" @@ -102,6 +103,48 @@ func TestProcessNegativeTickBudgetRunsStepToCompletion(t *testing.T) { } } +func TestProcessTickBudgetBoundaries(t *testing.T) { + const script = ` + local s = 0 + for i = 1, 100000 do s = s + i end + return s + ` + t.Run("max int64 runs the step to completion", func(t *testing.T) { + proc := initPreemptProcess(t, script, math.MaxInt64) + var output process.StepOutput + if err := proc.Step(nil, &output); err != nil { + t.Fatal(err) + } + if output.Status() != process.StepDone { + t.Fatalf("expected StepDone in one step, got %v", output.Status()) + } + }) + t.Run("budget of one makes progress", func(t *testing.T) { + proc := initPreemptProcess(t, `return 7`, 1) + if v, _ := stepPreempted(t, proc, 100); lua.LVAsNumber(v) != 7 { + t.Fatalf("expected 7, got %v", v) + } + }) +} + +// A budget that runs out exactly as the last task ends neither loses the +// result nor reports a preemption with nothing left to run. +func TestProcessBudgetExhaustedAtTaskEnd(t *testing.T) { + const script = ` + local function f(n) return n + 1 end + local s = 0 + for i = 1, 5 do s = f(s) end + return s + ` + for budget := int64(1); budget <= 40; budget++ { + proc := initPreemptProcess(t, script, budget) + v, _ := stepPreempted(t, proc, 1000) + if lua.LVAsNumber(v) != 5 { + t.Fatalf("budget %d: expected 5, got %v", budget, v) + } + } +} + // The budget covers the whole step: coroutines that did not get to run // before it ran out are deferred to the next step rather than each getting a // fresh budget. From f6a6dbfe7b18eae92318ea0a4940760e5c07d834 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sat, 3 Oct 2026 10:34:20 -0400 Subject: [PATCH 30/50] test(lua): cover initializer retry and method lookup after initializers --- runtime/lua/engine/initializer_test.go | 78 ++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 runtime/lua/engine/initializer_test.go diff --git a/runtime/lua/engine/initializer_test.go b/runtime/lua/engine/initializer_test.go new file mode 100644 index 000000000..2503dacd6 --- /dev/null +++ b/runtime/lua/engine/initializer_test.go @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: MPL-2.0 + +package engine + +import ( + "testing" + + lua "github.com/wippyai/go-lua" + "github.com/wippyai/runtime/api/process" +) + +// An initializer that fails is retried by the next execution of the process; +// completion callbacks run once, after the last initializer succeeds. +func TestFailedInitializerIsRetriedByNextExecution(t *testing.T) { + var attempts, done, initialized int + proc := mustNewProcess(t, + WithModuleBinder(wrapBinder(func(l *lua.LState) { + DeferInitializer(l, Initializer{ + Fn: l.NewFunction(func(l *lua.LState) int { + attempts++ + if attempts == 1 { + l.RaiseError("init failed") + } + l.Push(lua.LNumber(attempts)) + return 1 + }), + Done: func(lua.LValue) { done++ }, + }) + OnInitialized(l, func() { initialized++ }) + })), + WithScript(`return { main = function() return 1 end }`, "m.lua")) + t.Cleanup(proc.Close) + + if err := proc.Init(frameContext(), "main", nil); err != nil { + t.Fatal(err) + } + var output process.StepOutput + if err := proc.Step(nil, &output); err == nil { + t.Fatal("expected the failing initializer to fail the step") + } + if done != 0 || initialized != 0 { + t.Fatalf("failed initializer completed: done=%d initialized=%d", done, initialized) + } + + runProcess(t, proc, 1) + if attempts != 2 || done != 1 || initialized != 1 { + t.Fatalf("attempts=%d done=%d initialized=%d", attempts, done, initialized) + } +} + +// A missing method fails with a not-found error after the initializers +// completed, and a later execution does not run them again. +func TestMethodNotFoundAfterInitializers(t *testing.T) { + var runs, done int + proc := mustNewProcess(t, + WithModuleBinder(wrapBinder(func(l *lua.LState) { + DeferInitializer(l, Initializer{ + Fn: l.NewFunction(func(*lua.LState) int { runs++; return 0 }), + Done: func(lua.LValue) { done++ }, + }) + })), + WithScript(`return { main = function() return 1 end }`, "m.lua")) + t.Cleanup(proc.Close) + + if err := proc.Init(frameContext(), "missing", nil); err != nil { + t.Fatal(err) + } + var output process.StepOutput + requireMethodNotFound(t, proc.Step(nil, &output)) + if runs != 1 || done != 1 { + t.Fatalf("initializer runs=%d done=%d, want 1 and 1", runs, done) + } + + runProcess(t, proc, 1) + if runs != 1 { + t.Fatalf("completed initializer ran again: %d", runs) + } +} From 814868691e315971705216a4cfa784b7ae9797f5 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sat, 3 Oct 2026 10:35:20 -0400 Subject: [PATCH 31/50] test(actor): cover InjectQueue.IsEmpty --- system/scheduler/actor/inject_test.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 system/scheduler/actor/inject_test.go diff --git a/system/scheduler/actor/inject_test.go b/system/scheduler/actor/inject_test.go new file mode 100644 index 000000000..9e7da6e29 --- /dev/null +++ b/system/scheduler/actor/inject_test.go @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: MPL-2.0 + +package actor + +import "testing" + +func TestInjectQueueIsEmpty(t *testing.T) { + q := NewInjectQueue() + if !q.IsEmpty() { + t.Fatal("a new queue is empty") + } + q.Push(&Processor{}) + if q.IsEmpty() { + t.Fatal("a queue holding a processor is not empty") + } + if q.Pop() == nil { + t.Fatal("expected the queued processor") + } + if !q.IsEmpty() { + t.Fatal("a drained queue is empty") + } +} From 126e1b1d8a085307e2bbba5d6c8f30fa7135ca11 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sat, 3 Oct 2026 10:47:28 -0400 Subject: [PATCH 32/50] fix(api): reject non-map meta.options and accept nil execution options A present meta.options of another shape registered with default budgets, and ExecutionBudgetsFromOptions dereferenced a nil interface. --- api/runtime/lua/options.go | 11 +++++++++-- api/runtime/lua/options_test.go | 18 ++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/api/runtime/lua/options.go b/api/runtime/lua/options.go index dae6f294e..30552c91d 100644 --- a/api/runtime/lua/options.go +++ b/api/runtime/lua/options.go @@ -48,9 +48,13 @@ func (b ExecutionBudgets) Override(o ExecutionBudgets) ExecutionBudgets { } // ExecutionBudgetsFromOptions reads the execution options from an options -// bag. subject names the source in error messages. +// bag; nil options hold no execution options. subject names the source in +// error messages. func ExecutionBudgetsFromOptions(options attrs.Attributes, subject string) (ExecutionBudgets, error) { var b ExecutionBudgets + if options == nil { + return b, nil + } if raw, ok := options.Get(ProcessOptionTickBudget); ok && raw != nil { v, err := optionInt64(raw) if err != nil { @@ -69,7 +73,7 @@ func ExecutionBudgetsFromOptions(options attrs.Attributes, subject string) (Exec } // EntryExecutionBudgets reads the execution options of a process entry from -// meta.options. The options are only accepted nested under meta.options. +// meta.options, which must be a map when present. The options are only accepted nested under meta.options. func EntryExecutionBudgets(meta attrs.Bag) (ExecutionBudgets, error) { for _, key := range [...]string{ProcessOptionTickBudget, ProcessOptionMaxSteps} { if _, ok := meta.Get(key); ok { @@ -78,6 +82,9 @@ func EntryExecutionBudgets(meta attrs.Bag) (ExecutionBudgets, error) { } options, ok := meta.GetBag(entryOptionsMetaKey) if !ok { + if raw, present := meta.Get(entryOptionsMetaKey); present && raw != nil { + return ExecutionBudgets{}, fmt.Errorf("process entry meta.%s must be a map, got %T", entryOptionsMetaKey, raw) + } return ExecutionBudgets{}, nil } return ExecutionBudgetsFromOptions(options, "process entry meta.options") diff --git a/api/runtime/lua/options_test.go b/api/runtime/lua/options_test.go index 5238b803c..9a4dafc49 100644 --- a/api/runtime/lua/options_test.go +++ b/api/runtime/lua/options_test.go @@ -70,3 +70,21 @@ func TestExecutionBudgetsOverride(t *testing.T) { t.Fatalf("unexpected override %+v", got) } } + +func TestEntryExecutionBudgetsRejectsNonMapOptions(t *testing.T) { + for _, raw := range []any{"fast", []any{"tick_budget"}, 5} { + if _, err := EntryExecutionBudgets(attrs.Bag{"options": raw}); err == nil { + t.Fatalf("meta.options %T must be rejected", raw) + } + } + if b, err := EntryExecutionBudgets(attrs.Bag{"options": nil}); err != nil || b.TickBudgetSet || b.MaxStepsSet { + t.Fatalf("null meta.options holds no options, got %+v %v", b, err) + } +} + +func TestExecutionBudgetsFromNilOptionsIsEmpty(t *testing.T) { + b, err := ExecutionBudgetsFromOptions(nil, "test") + if err != nil || b.TickBudgetSet || b.MaxStepsSet { + t.Fatalf("expected empty budgets, got %+v %v", b, err) + } +} From 73302e62b2e116da958b53c8e8ef8d4ab57bbf89 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sat, 3 Oct 2026 10:50:19 -0400 Subject: [PATCH 33/50] fix(actor): start a new yield epoch when an upgrade replaces the process Yield tags restart in the replacement, so a completion of the replaced code could resume an unrelated task. EventQueue.RetireYieldCompletions makes earlier completers inert and drops queued completions; messages and wakes keep the queue generation. --- api/process/queue.go | 45 ++++++-- api/process/queue_generation_test.go | 19 ++++ system/scheduler/actor/upgrade_yield_test.go | 110 +++++++++++++++++++ system/scheduler/actor/worker.go | 5 + 4 files changed, 172 insertions(+), 7 deletions(-) create mode 100644 system/scheduler/actor/upgrade_yield_test.go diff --git a/api/process/queue.go b/api/process/queue.go index fd32b263c..5ee594d27 100644 --- a/api/process/queue.go +++ b/api/process/queue.go @@ -40,8 +40,12 @@ type EventQueue struct { events []Event drainBuf []Event generation atomic.Uint64 - mu sync.Mutex - closed atomic.Bool + // yieldEpoch identifies the code incarnation whose yield tags are valid. + // It changes when a process is replaced in place, so completions of the + // replaced code cannot resume the replacement. + yieldEpoch atomic.Uint64 + mu sync.Mutex + closed atomic.Bool } // messageTopicState is the accounting identity for one bounded topic @@ -120,6 +124,12 @@ func (q *EventQueue) Push(e Event, gen uint64) bool { // PushWithError preserves admission errors such as a full bounded mailbox. func (q *EventQueue) PushWithError(e Event, gen uint64) error { + return q.push(e, gen, nil) +} + +// push admits e when the generation, and the yield epoch if one is given, +// still match. +func (q *EventQueue) push(e Event, gen uint64, epoch *uint64) error { // Fast path: check generation and closed without lock if q.generation.Load() != gen { return ErrProcessClosed @@ -130,7 +140,7 @@ func (q *EventQueue) PushWithError(e Event, gen uint64) error { q.mu.Lock() // Recheck under lock - if q.generation.Load() != gen || q.closed.Load() { + if q.generation.Load() != gen || q.closed.Load() || (epoch != nil && q.yieldEpoch.Load() != *epoch) { q.mu.Unlock() return ErrProcessClosed } @@ -147,6 +157,24 @@ func (q *EventQueue) PushWithError(e Event, gen uint64) error { return nil } +// RetireYieldCompletions starts a new yield epoch for a process replaced in +// place: completers created before the call become inert and yield +// completions already queued are discarded. Messages and senders bound to the +// queue generation are unaffected. +func (q *EventQueue) RetireYieldCompletions() { + q.mu.Lock() + q.yieldEpoch.Add(1) + kept := q.events[:0] + for _, e := range q.events { + if e.Type != EventYieldComplete { + kept = append(kept, e) + } + } + clear(q.events[len(kept):]) + q.events = kept + q.mu.Unlock() +} + // SetAdmission installs an execution's ingress policy before publishing its // queue to senders. Reset removes the old policy before a queue is reused. func (q *EventQueue) SetAdmission(admission EventAdmission) { @@ -545,20 +573,23 @@ type YieldScheduler interface { } // YieldCompleter delivers yield completion results to a queue. -// Bound to a specific generation - stale completers silently no-op. +// Bound to a specific generation and yield epoch - stale completers silently no-op. // This breaks the direct reference to Processor, preventing races // when processor is released to pool while handler goroutines are still running. type YieldCompleter struct { scheduler YieldScheduler queue *EventQueue gen uint64 + epoch uint64 } -// NewYieldCompleter creates a completer bound to current queue generation. +// NewYieldCompleter creates a completer bound to the current queue generation +// and yield epoch. func (q *EventQueue) NewYieldCompleter(sched YieldScheduler) *YieldCompleter { return &YieldCompleter{ queue: q, gen: q.generation.Load(), + epoch: q.yieldEpoch.Load(), scheduler: sched, } } @@ -566,12 +597,12 @@ func (q *EventQueue) NewYieldCompleter(sched YieldScheduler) *YieldCompleter { // CompleteYield implements dispatcher.ResultReceiver. // Safe to call from any goroutine - uses generation to detect staleness. func (c *YieldCompleter) CompleteYield(tag uint64, data any, err error) { - if !c.queue.Push(Event{ + if c.queue.push(Event{ Type: EventYieldComplete, Tag: tag, Data: data, Error: err, - }, c.gen) { + }, c.gen, &c.epoch) != nil { return } diff --git a/api/process/queue_generation_test.go b/api/process/queue_generation_test.go index 2f4bec724..775d50a32 100644 --- a/api/process/queue_generation_test.go +++ b/api/process/queue_generation_test.go @@ -100,3 +100,22 @@ func TestA05ResetDrainsStaleSignal(t *testing.T) { default: } } + +func TestRetireYieldCompletionsInvalidatesEarlierCompleters(t *testing.T) { + queue := NewEventQueue() + wake := &boundaryWakeRecorder{} + old := queue.NewYieldCompleter(wake) + old.CompleteYield(1, "queued-before", nil) + require.True(t, queue.PushDirect(Event{Type: EventMessage, Data: "message"})) + + queue.RetireYieldCompletions() + fresh := queue.NewYieldCompleter(wake) + old.CompleteYield(1, "stale", nil) + fresh.CompleteYield(1, "fresh", nil) + + events := queue.Drain() + require.Len(t, events, 2) + assert.Equal(t, EventMessage, events[0].Type, "messages survive") + assert.Equal(t, "fresh", events[1].Data) + assert.Equal(t, 2, wake.calls, "the retired completer does not wake") +} diff --git a/system/scheduler/actor/upgrade_yield_test.go b/system/scheduler/actor/upgrade_yield_test.go new file mode 100644 index 000000000..eff117a6a --- /dev/null +++ b/system/scheduler/actor/upgrade_yield_test.go @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: MPL-2.0 + +package actor + +import ( + "context" + "testing" + "time" + + ctxapi "github.com/wippyai/runtime/api/context" + "github.com/wippyai/runtime/api/dispatcher" + "github.com/wippyai/runtime/api/payload" + pidapi "github.com/wippyai/runtime/api/pid" + "github.com/wippyai/runtime/api/process" + "github.com/wippyai/runtime/api/registry" + "github.com/wippyai/runtime/api/relay" + "github.com/wippyai/runtime/system/scheduler" +) + +// yieldingProcess yields tag 1 on its first step. Its second step either +// requests an upgrade or, when it has no request, records yield completions. +type yieldingProcess struct { + upgrade *process.UpgradeRequest + completed chan any + steps int +} + +func (*yieldingProcess) Init(context.Context, string, payload.Payloads) error { return nil } +func (p *yieldingProcess) Step(events []process.Event, out *process.StepOutput) error { + p.steps++ + if p.steps == 1 { + out.Yield(YieldCmd{}, 1) + out.WaitForYields() + return nil + } + for _, e := range events { + if e.Type == process.EventYieldComplete { + p.completed <- e.Data + out.Done(nil) + return nil + } + } + if p.upgrade != nil && len(events) > 0 { + out.SetUpgrade(p.upgrade) + return nil + } + out.WaitForYields() + return nil +} +func (*yieldingProcess) Send(*relay.Package) error { return nil } +func (*yieldingProcess) Close() {} + +// A yield completion of the replaced code never reaches the replacement, even +// when its tag matches a yield of the replacement; the replacement's own +// completion is delivered. +func TestUpgradeRetiresYieldCompletionsOfReplacedCode(t *testing.T) { + receivers := make(chan dispatcher.ResultReceiver, 2) + reg := scheduler.NewRegistry() + reg.Register(CmdYield, dispatcher.HandlerFunc(func(_ context.Context, _ dispatcher.Command, _ uint64, r dispatcher.ResultReceiver) error { + receivers <- r + return nil + })) + reg.Register(CmdComplete, CompleteHandler()) + sched := NewScheduler(reg, WithWorkers(1), WithLifecycle(&testLifecycle{})) + sched.Start() + defer testStopScheduler(sched) + + completed := make(chan any, 2) + next := registry.NewID("app", "next") + factory := &mockFactory{createFunc: func(registry.ID) (process.Process, *process.Meta, error) { + return &yieldingProcess{completed: completed}, &process.Meta{Method: "main"}, nil + }} + appCtx := ctxapi.WithAppContext(context.Background(), ctxapi.NewAppContext()) + process.WithFactory(appCtx, factory) + frameCtx, fc := ctxapi.OpenFrameContext(appCtx) + fc.Seal() + id := pidapi.PID{UniqID: "yielder"} + source := &yieldingProcess{upgrade: &process.UpgradeRequest{Source: next}, completed: completed} + if _, err := sched.Submit(frameCtx, id, source, "", nil); err != nil { + t.Fatal(err) + } + oldReceiver := receive(t, receivers, "source yield") + if err := sched.Send(&relay.Package{Target: id}); err != nil { + t.Fatal(err) + } + newReceiver := receive(t, receivers, "replacement yield") + + oldReceiver.CompleteYield(1, "stale", nil) + newReceiver.CompleteYield(1, "fresh", nil) + + select { + case got := <-completed: + if got != "fresh" { + t.Fatalf("replacement received %v, want its own completion", got) + } + case <-time.After(5 * time.Second): + t.Fatal("replacement did not receive its completion") + } +} + +func receive(t *testing.T, ch <-chan dispatcher.ResultReceiver, what string) dispatcher.ResultReceiver { + t.Helper() + select { + case r := <-ch: + return r + case <-time.After(5 * time.Second): + t.Fatalf("%s was not dispatched", what) + return nil + } +} diff --git a/system/scheduler/actor/worker.go b/system/scheduler/actor/worker.go index 68c6d8f5b..1a3145344 100644 --- a/system/scheduler/actor/worker.go +++ b/system/scheduler/actor/worker.go @@ -530,6 +530,11 @@ func (w *Worker) executeOne(proc *Processor) { // Swap proc.Process = newProc enablePreemption(newProc) + // The replacement numbers its yields from the start; completions of the + // replaced code must not resume them. Messages and wakes stay bound to + // the unchanged queue generation. + proc.queue.RetireYieldCompletions() + proc.completer = proc.queue.NewYieldCompleter(w.scheduler) // Init new process method := "main" From 3dfc124aebb7170902c52dbc3f5795a828cf839e Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sat, 3 Oct 2026 10:51:16 -0400 Subject: [PATCH 34/50] fix(actor): make a wake's generation check atomic with its state transition A completion or message bound to an earlier incarnation could pass the generation check, then wake or flag the incarnation that reused the slot, costing it a spurious step. The check and the transition now run under one per-processor lock that slot reuse also takes. --- system/scheduler/actor/processor.go | 35 +++++++++++++ system/scheduler/actor/scheduler.go | 22 ++------- system/scheduler/actor/wake_reuse_test.go | 60 +++++++++++++++++++++++ 3 files changed, 99 insertions(+), 18 deletions(-) create mode 100644 system/scheduler/actor/wake_reuse_test.go diff --git a/system/scheduler/actor/processor.go b/system/scheduler/actor/processor.go index 09573194a..997bc12fa 100644 --- a/system/scheduler/actor/processor.go +++ b/system/scheduler/actor/processor.go @@ -93,6 +93,7 @@ type Processor struct { output process.StepOutput gen atomic.Uint64 steps atomic.Uint64 + wakeMu sync.Mutex id uint64 startedAt int64 state atomic.Int32 @@ -100,6 +101,40 @@ type Processor struct { pooled bool } +// setGeneration binds the processor to a new queue generation. A wake in +// progress completes against the incarnation it checked, so a wake bound to an +// earlier generation never transitions this one. +func (p *Processor) setGeneration(gen uint64) { + p.wakeMu.Lock() + p.gen.Store(gen) + p.wakeMu.Unlock() +} + +// casStateAt is casState for a wake bound to a queue generation: the +// transition happens only while the processor still runs that generation. +func (p *Processor) casStateAt(gen uint64, old, newState ProcessState) bool { + p.wakeMu.Lock() + defer p.wakeMu.Unlock() + return p.gen.Load() == gen && p.casState(old, newState) +} + +// wake makes a processor that waits runnable once an event was queued for it +// under gen, and reports whether the caller must schedule it. A running +// processor is flagged instead, and re-queues itself when it stops. The +// generation check and the transition are one step against slot reuse. +func (p *Processor) wake(gen uint64) bool { + p.wakeMu.Lock() + defer p.wakeMu.Unlock() + if p.gen.Load() != gen { + return false + } + if p.casState(StateBlocked, StateReady) || p.casState(StateIdle, StateReady) { + return true + } + p.setWakeup(StateRunning) + return false +} + // publishSignalRef publishes immutable identity, generation and the // incarnation's context cancel for message delivery, termination and // lifecycle scans. Source is optional; PID identity is not. diff --git a/system/scheduler/actor/scheduler.go b/system/scheduler/actor/scheduler.go index c3c48f90e..234fb9312 100644 --- a/system/scheduler/actor/scheduler.go +++ b/system/scheduler/actor/scheduler.go @@ -283,21 +283,9 @@ func (s *Scheduler) WakeProcessor(q *process.EventQueue, gen uint64) { } proc := v.(*Processor) - // Verify generation matches to avoid waking wrong processor - if proc.gen.Load() != gen { - return - } - - // Same wake logic as processor.CompleteYield - if proc.casState(StateBlocked, StateReady) { - s.injectOrGlobal(proc) - return - } - if proc.casState(StateIdle, StateReady) { + if proc.wake(gen) { s.injectOrGlobal(proc) - return } - proc.setWakeup(StateRunning) } func (s *Scheduler) Submit(ctx context.Context, pid pid.PID, p process.Process, method string, input payload.Payloads) (*Processor, error) { @@ -345,7 +333,7 @@ func (s *Scheduler) Submit(ctx context.Context, pid pid.PID, p process.Process, if admission, ok := p.(interface{ EventAdmission() process.EventAdmission }); ok { proc.queue.SetAdmission(admission.EventAdmission()) } - proc.gen.Store(proc.queue.Generation()) + proc.setGeneration(proc.queue.Generation()) proc.completer = proc.queue.NewYieldCompleter(s) proc.publishSignalRef(cancel) proc.publishInspectorRef() @@ -481,7 +469,7 @@ func (s *Scheduler) CreateProcessor(ctx context.Context, pid pid.PID, p process. // Reset queue for this execution and cache generation proc.queue.Reset() - proc.gen.Store(proc.queue.Generation()) + proc.setGeneration(proc.queue.Generation()) proc.completer = proc.queue.NewYieldCompleter(s) proc.publishSignalRef(cancel) proc.publishInspectorRef() @@ -612,9 +600,7 @@ func (s *Scheduler) deliverToProcError(proc *Processor, gen uint64, pkg *relay.P // Wake process if waiting for messages. // CAS ensures exactly-once wake even with concurrent senders. // Try both Idle (waiting on select) and Blocked (waiting on yield completion). - if proc.casState(StateIdle, StateReady) { - s.injectOrGlobal(proc) - } else if proc.casState(StateBlocked, StateReady) { + if proc.casStateAt(gen, StateIdle, StateReady) || proc.casStateAt(gen, StateBlocked, StateReady) { s.injectOrGlobal(proc) } diff --git a/system/scheduler/actor/wake_reuse_test.go b/system/scheduler/actor/wake_reuse_test.go new file mode 100644 index 000000000..2f023fbd4 --- /dev/null +++ b/system/scheduler/actor/wake_reuse_test.go @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: MPL-2.0 + +package actor + +import ( + "sync" + "testing" + + "github.com/wippyai/runtime/api/process" +) + +// A wake bound to an earlier incarnation of a processor slot never changes the +// state of the incarnation that reuses the slot, whichever way the reuse +// interleaves with the wake. +func TestStaleWakeDoesNotTransitionReusedSlot(t *testing.T) { + s := NewScheduler(nil) + proc := &Processor{queue: process.NewEventQueue()} + s.byQueue.Store(proc.queue, proc) + + const rounds = 20000 + for i := 0; i < rounds; i++ { + oldGen := uint64(2*i + 1) + proc.gen.Store(oldGen) + proc.state.Store(int32(StateBlocked)) + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + s.WakeProcessor(proc.queue, oldGen) + }() + // The slot is reused: the next incarnation publishes its generation and + // blocks. + proc.setGeneration(oldGen + 1) + proc.state.Store(int32(StateBlocked)) + wg.Wait() + + if got := ProcessState(proc.state.Load()) & stateMask; got != StateBlocked { + t.Fatalf("round %d: stale wake moved the reused slot to %s", i, StateName(got)) + } + for s.global.Pop() != nil { + } + } +} + +func TestStaleWakeLeavesRunningSlotUnflagged(t *testing.T) { + proc := &Processor{queue: process.NewEventQueue()} + proc.setGeneration(2) + proc.state.Store(int32(StateRunning)) + + if proc.wake(1) { + t.Fatal("a stale wake must not schedule the processor") + } + if got := ProcessState(proc.state.Load()); got != StateRunning { + t.Fatalf("stale wake changed the state to %d", got) + } + if proc.wake(2); ProcessState(proc.state.Load()) != StateRunning|wakeupFlag { + t.Fatalf("a current wake flags the running processor, state %d", proc.state.Load()) + } +} From 0ff9f4c97a91c15557cb19b6643d9648168a000c Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sat, 3 Oct 2026 10:52:27 -0400 Subject: [PATCH 35/50] fix(actor): wake a parked worker for newly runnable work wakeAny notified the first worker whose token was free, which could be the worker running the step that queued the work. A worker that then blocked in a long step left the work stranded while another worker stayed parked. Workers publish whether they are parked and only those are notified. --- system/scheduler/actor/preempt_test.go | 18 ++++++++++++ system/scheduler/actor/scheduler.go | 6 +++- system/scheduler/actor/worker.go | 38 ++++++++++++++++---------- 3 files changed, 46 insertions(+), 16 deletions(-) diff --git a/system/scheduler/actor/preempt_test.go b/system/scheduler/actor/preempt_test.go index fa5c18f18..b8ee86243 100644 --- a/system/scheduler/actor/preempt_test.go +++ b/system/scheduler/actor/preempt_test.go @@ -351,3 +351,21 @@ func TestUpgradeCarriesStepCount(t *testing.T) { t.Fatalf("expected 41 steps carried to the replacement, got %d", got) } } + +// A wake-up goes to a worker that is parked: a worker that is running a step +// cannot take the work before that step ends. +func TestWakeAnySignalsParkedWorker(t *testing.T) { + s := newPreemptTestScheduler(2, &testLifecycle{}) + busy, parked := newWorker(0, s), newWorker(1, s) + s.storeWorkers([]*Worker{busy, parked}) + parked.parked.Store(true) + + s.wakeAny() + + if busy.notified.Load() { + t.Fatal("a running worker was notified while another worker was parked") + } + if !parked.notified.Load() { + t.Fatal("the parked worker was not notified") + } +} diff --git a/system/scheduler/actor/scheduler.go b/system/scheduler/actor/scheduler.go index 234fb9312..a083a6426 100644 --- a/system/scheduler/actor/scheduler.go +++ b/system/scheduler/actor/scheduler.go @@ -230,9 +230,13 @@ func (s *Scheduler) Stop(ctx context.Context) { }) } +// wakeAny notifies one parked worker. A worker that is not parked looks for +// work when its current step ends, so notifying it cannot start the work +// while that step runs; the park protocol guarantees that a worker about to +// park either sees the queued work or is seen as parked. func (s *Scheduler) wakeAny() { for _, w := range s.workerSnapshot() { - if w.signal() { + if w.parked.Load() && w.signal() { return } } diff --git a/system/scheduler/actor/worker.go b/system/scheduler/actor/worker.go index 1a3145344..6c9a0f5a6 100644 --- a/system/scheduler/actor/worker.go +++ b/system/scheduler/actor/worker.go @@ -17,18 +17,20 @@ import ( ) type Worker struct { - batchBuf [32]*Processor - local *Deque - inject *InjectQueue - scheduler *Scheduler - parkCond *sync.Cond - done chan struct{} - parkMu sync.Mutex - routeMu sync.Mutex - id int - executed atomic.Uint64 - stolen atomic.Uint64 - notified atomic.Bool + batchBuf [32]*Processor + local *Deque + inject *InjectQueue + scheduler *Scheduler + parkCond *sync.Cond + done chan struct{} + parkMu sync.Mutex + routeMu sync.Mutex + id int + executed atomic.Uint64 + stolen atomic.Uint64 + notified atomic.Bool + // parked is set while the worker is counted in Scheduler.parked. + parked atomic.Bool executing bool // guarded by routeMu retiring atomic.Bool dispatchesSinceFair uint8 @@ -182,21 +184,27 @@ func (w *Worker) requeuePreempted(proc *Processor) { } } +func (w *Worker) unpark() { + w.parked.Store(false) + w.scheduler.parked.Add(-1) +} + func (w *Worker) park() { s := w.scheduler // A worker counts as parked from before its last work check until it // stops waiting, so a producer that publishes work and then reads the // count either sees the worker or the worker sees the work. s.parked.Add(1) + w.parked.Store(true) w.parkMu.Lock() for { if w.retiring.Load() { - s.parked.Add(-1) + w.unpark() w.parkMu.Unlock() return } if proc := w.findWork(); proc != nil { - s.parked.Add(-1) + w.unpark() shouldWake := s.global.Len() > 0 w.parkMu.Unlock() if shouldWake { @@ -207,7 +215,7 @@ func (w *Worker) park() { return } if s.phase.Load() == phaseStoppingWorkers { - s.parked.Add(-1) + w.unpark() w.parkMu.Unlock() return } From f0facbe229183a9fe3c56949de09b2de7dc71b5e Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sat, 3 Oct 2026 10:53:52 -0400 Subject: [PATCH 36/50] fix(actor): Stop waits for processors whose admission is in progress Submit published a processor to the routing maps before its admission ended, so Stop could cancel and complete it while Submit still finished initialization and published it Ready. Admissions are now registered under the stopping check and Stop waits for them before it acts. --- system/scheduler/actor/scheduler.go | 86 ++++++++++++++----- system/scheduler/actor/stop_admission_test.go | 74 ++++++++++++++++ 2 files changed, 138 insertions(+), 22 deletions(-) create mode 100644 system/scheduler/actor/stop_admission_test.go diff --git a/system/scheduler/actor/scheduler.go b/system/scheduler/actor/scheduler.go index a083a6426..92ed0a986 100644 --- a/system/scheduler/actor/scheduler.go +++ b/system/scheduler/actor/scheduler.go @@ -95,26 +95,31 @@ func WithMaxProcesses(maxProcs int64) Option { } type Scheduler struct { - lifecycle process.Lifecycle - topology topology.Topology - registry dispatcher.Registry - global *Queue - drainCh chan struct{} - byQueue sync.Map - byPID sync.Map - workers atomic.Pointer[workerSet] - pinSet affinity.Set - wg sync.WaitGroup - controlMu sync.Mutex - initialWorkers int - maxProcesses int64 - localQueueSize int - processorCount atomic.Int64 - retiredExecuted atomic.Uint64 - retiredStolen atomic.Uint64 - queueSize int - nextID atomic.Uint64 - parked atomic.Int32 + lifecycle process.Lifecycle + topology topology.Topology + registry dispatcher.Registry + global *Queue + drainCh chan struct{} + byQueue sync.Map + byPID sync.Map + workers atomic.Pointer[workerSet] + pinSet affinity.Set + wg sync.WaitGroup + controlMu sync.Mutex + initialWorkers int + maxProcesses int64 + localQueueSize int + processorCount atomic.Int64 + retiredExecuted atomic.Uint64 + retiredStolen atomic.Uint64 + queueSize int + nextID atomic.Uint64 + parked atomic.Int32 + // admitting counts Submit and CreateProcessor calls that have not + // finished publishing their processor; guarded by admitMu. + admitting int + admitMu sync.Mutex + admitIdle *sync.Cond phase atomic.Uint32 collectStats atomic.Bool started bool @@ -123,6 +128,39 @@ type Scheduler struct { func (s *Scheduler) isStopping() bool { return s.phase.Load() != phaseRunning } +// beginAdmission registers an admission and reports false once the scheduler +// is stopping. The check and the registration are one step, so Stop's +// awaitAdmissions sees every admission that was allowed to begin. +func (s *Scheduler) beginAdmission() bool { + s.admitMu.Lock() + defer s.admitMu.Unlock() + if s.isStopping() { + return false + } + s.admitting++ + return true +} + +func (s *Scheduler) endAdmission() { + s.admitMu.Lock() + s.admitting-- + if s.admitting == 0 { + s.admitIdle.Broadcast() + } + s.admitMu.Unlock() +} + +// awaitAdmissions blocks until every admission that began before the +// scheduler started stopping has published its processor or failed, so Stop +// only acts on fully admitted processors. +func (s *Scheduler) awaitAdmissions() { + s.admitMu.Lock() + for s.admitting > 0 { + s.admitIdle.Wait() + } + s.admitMu.Unlock() +} + func NewScheduler(registry dispatcher.Registry, opts ...Option) *Scheduler { s := &Scheduler{ registry: registry, @@ -137,6 +175,7 @@ func NewScheduler(registry dispatcher.Registry, opts ...Option) *Scheduler { s.global = NewQueue(s.queueSize) s.drainCh = make(chan struct{}, 1) + s.admitIdle = sync.NewCond(&s.admitMu) workers := make([]*Worker, s.initialWorkers) for i := range workers { workers[i] = newWorker(i, s) @@ -161,6 +200,7 @@ func (s *Scheduler) Stop(ctx context.Context) { return } s.controlMu.Unlock() + s.awaitAdmissions() // Push cancel event directly to each processor's queue. // Safe because draining prevents pool release. @@ -293,9 +333,10 @@ func (s *Scheduler) WakeProcessor(q *process.EventQueue, gen uint64) { } func (s *Scheduler) Submit(ctx context.Context, pid pid.PID, p process.Process, method string, input payload.Payloads) (*Processor, error) { - if s.isStopping() { + if !s.beginAdmission() { return nil, process.ErrSchedulerStopping } + defer s.endAdmission() if s.maxProcesses > 0 && s.processorCount.Load() >= s.maxProcesses { return nil, process.ErrMaxProcessesExceeded } @@ -448,9 +489,10 @@ func (s *Scheduler) finishProcessor(proc *Processor, result *process.StepOutput, } func (s *Scheduler) CreateProcessor(ctx context.Context, pid pid.PID, p process.Process) (*Processor, error) { - if s.isStopping() { + if !s.beginAdmission() { return nil, process.ErrSchedulerStopping } + defer s.endAdmission() if s.maxProcesses > 0 && s.processorCount.Load() >= s.maxProcesses { return nil, process.ErrMaxProcessesExceeded } diff --git a/system/scheduler/actor/stop_admission_test.go b/system/scheduler/actor/stop_admission_test.go new file mode 100644 index 000000000..4607d6ce7 --- /dev/null +++ b/system/scheduler/actor/stop_admission_test.go @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: MPL-2.0 + +package actor + +import ( + "context" + "slices" + "sync" + "testing" + "time" + + pidapi "github.com/wippyai/runtime/api/pid" + "github.com/wippyai/runtime/api/process" + "github.com/wippyai/runtime/api/runtime" +) + +// Stop does not complete a processor whose admission is still in progress: +// the processor finishes admission, is published, and stops like any other. +func TestStopWaitsForProcessorsBeingAdmitted(t *testing.T) { + var mu sync.Mutex + var events []string + record := func(e string) { + mu.Lock() + events = append(events, e) + mu.Unlock() + } + entered, release := make(chan struct{}), make(chan struct{}) + lc := &testLifecycle{ + onStart: func(context.Context, pidapi.PID, process.Process) { + close(entered) + <-release + record("start returned") + }, + onComplete: func(context.Context, pidapi.PID, *runtime.Result) { record("complete") }, + } + sched := newPreemptTestScheduler(1, lc) + sched.Start() + + p := &closeCountingProcess{} + submitted := make(chan error, 1) + go func() { + _, err := sched.Submit(context.Background(), pidapi.PID{UniqID: "admitting"}, p, "", nil) + submitted <- err + }() + <-entered + + stopped := make(chan struct{}) + go func() { + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + sched.Stop(ctx) + close(stopped) + }() + // Without a wait for admission Stop returns once its deadline passes. + select { + case <-stopped: + case <-time.After(600 * time.Millisecond): + } + close(release) + + if err := <-submitted; err != nil { + t.Fatalf("submit: %v", err) + } + <-stopped + + mu.Lock() + defer mu.Unlock() + if want := []string{"start returned", "complete"}; !slices.Equal(events, want) { + t.Fatalf("lifecycle events %v, want %v", events, want) + } + if got := p.closed.Load(); got != 1 { + t.Fatalf("process closed %d times", got) + } +} From 6d13c23b4c05e7f7c3a1323af8ccc47b79f96372 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sat, 3 Oct 2026 10:54:13 -0400 Subject: [PATCH 37/50] test(actor): assert a stale yield completion leaves the next process untouched --- .../scheduler/actor/close_ownership_test.go | 43 +++++++++++-------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/system/scheduler/actor/close_ownership_test.go b/system/scheduler/actor/close_ownership_test.go index d83779935..20956d85e 100644 --- a/system/scheduler/actor/close_ownership_test.go +++ b/system/scheduler/actor/close_ownership_test.go @@ -142,18 +142,15 @@ func (*waitingProcess) Close() {} // incarnation is dropped by the queue (TestA03StaleCompletionDoesNotEnqueue) // instead of reaching the processor's next user. func TestStaleYieldCompletionDoesNotWakeReusedProcessor(t *testing.T) { - received := make(chan dispatcher.ResultReceiver, 1) + receivers := make(chan dispatcher.ResultReceiver, 2) reg := scheduler.NewRegistry() reg.Register(CmdYield, dispatcher.HandlerFunc(func(_ context.Context, _ dispatcher.Command, _ uint64, r dispatcher.ResultReceiver) error { - select { - case received <- r: - default: - } + receivers <- r return nil })) reg.Register(CmdComplete, CompleteHandler()) - done := make(chan struct{}, 1) - lc := &testLifecycle{onComplete: func(context.Context, pidapi.PID, *apiruntime.Result) { done <- struct{}{} }} + completions := make(chan string, 2) + lc := &testLifecycle{onComplete: func(_ context.Context, id pidapi.PID, _ *apiruntime.Result) { completions <- id.UniqID }} sched := NewScheduler(reg, WithWorkers(1), WithLifecycle(lc)) sched.Start() defer testStopScheduler(sched) @@ -162,20 +159,28 @@ func TestStaleYieldCompletionDoesNotWakeReusedProcessor(t *testing.T) { if _, err := sched.Submit(ctx, pidapi.PID{UniqID: "a"}, &waitingProcess{}, "", nil); err != nil { t.Fatal(err) } - var receiver dispatcher.ResultReceiver - select { - case receiver = <-received: - case <-time.After(5 * time.Second): - t.Fatal("yield was not dispatched") - } - completer, ok := receiver.(*process.YieldCompleter) + stale, ok := receive(t, receivers, "first yield").(*process.YieldCompleter) if !ok { - t.Fatalf("handlers complete through the incarnation's completer, got %T", receiver) + t.Fatal("handlers complete through the incarnation's completer") } cancel() - <-done + if got := <-completions; got != "a" { + t.Fatalf("completed %q, want a", got) + } - // After completion the processor may serve another process; the old - // completer stays bound to the finished incarnation. - completer.CompleteYield(7, nil, nil) + // The next process, which may reuse the finished one's processor slot, + // waits on its own tag 7; the finished incarnation's completer is inert. + next := &waitingProcess{} + if _, err := sched.Submit(context.Background(), pidapi.PID{UniqID: "b"}, next, "", nil); err != nil { + t.Fatal(err) + } + fresh := receive(t, receivers, "second yield") + stale.CompleteYield(7, nil, nil) + fresh.CompleteYield(7, nil, nil) + if got := <-completions; got != "b" { + t.Fatalf("completed %q, want b", got) + } + if got := next.steps.Load(); got != 2 { + t.Fatalf("the next process took %d steps, want 2: the stale completion cost it a step", got) + } } From c694a184bbf78c2909198a659ed6733c0bca3f4a Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sat, 3 Oct 2026 10:55:03 -0400 Subject: [PATCH 38/50] style(actor): order scheduler and worker fields for alignment --- system/scheduler/actor/scheduler.go | 4 ++-- system/scheduler/actor/wake_reuse_test.go | 2 +- system/scheduler/actor/worker.go | 29 +++++++++++------------ 3 files changed, 17 insertions(+), 18 deletions(-) diff --git a/system/scheduler/actor/scheduler.go b/system/scheduler/actor/scheduler.go index 92ed0a986..bf55544d4 100644 --- a/system/scheduler/actor/scheduler.go +++ b/system/scheduler/actor/scheduler.go @@ -100,6 +100,7 @@ type Scheduler struct { registry dispatcher.Registry global *Queue drainCh chan struct{} + admitIdle *sync.Cond byQueue sync.Map byPID sync.Map workers atomic.Pointer[workerSet] @@ -117,9 +118,8 @@ type Scheduler struct { parked atomic.Int32 // admitting counts Submit and CreateProcessor calls that have not // finished publishing their processor; guarded by admitMu. - admitting int + admitting int32 admitMu sync.Mutex - admitIdle *sync.Cond phase atomic.Uint32 collectStats atomic.Bool started bool diff --git a/system/scheduler/actor/wake_reuse_test.go b/system/scheduler/actor/wake_reuse_test.go index 2f023fbd4..9ff7f5b8d 100644 --- a/system/scheduler/actor/wake_reuse_test.go +++ b/system/scheduler/actor/wake_reuse_test.go @@ -38,7 +38,7 @@ func TestStaleWakeDoesNotTransitionReusedSlot(t *testing.T) { if got := ProcessState(proc.state.Load()) & stateMask; got != StateBlocked { t.Fatalf("round %d: stale wake moved the reused slot to %s", i, StateName(got)) } - for s.global.Pop() != nil { + for s.global.Pop() != nil { //nolint:revive // drains the queue } } } diff --git a/system/scheduler/actor/worker.go b/system/scheduler/actor/worker.go index 6c9a0f5a6..fc003947a 100644 --- a/system/scheduler/actor/worker.go +++ b/system/scheduler/actor/worker.go @@ -17,22 +17,21 @@ import ( ) type Worker struct { - batchBuf [32]*Processor - local *Deque - inject *InjectQueue - scheduler *Scheduler - parkCond *sync.Cond - done chan struct{} - parkMu sync.Mutex - routeMu sync.Mutex - id int - executed atomic.Uint64 - stolen atomic.Uint64 - notified atomic.Bool - // parked is set while the worker is counted in Scheduler.parked. - parked atomic.Bool - executing bool // guarded by routeMu + batchBuf [32]*Processor + local *Deque + inject *InjectQueue + scheduler *Scheduler + parkCond *sync.Cond + done chan struct{} + parkMu sync.Mutex + routeMu sync.Mutex + id int + executed atomic.Uint64 + stolen atomic.Uint64 + notified atomic.Bool retiring atomic.Bool + parked atomic.Bool // set while counted in Scheduler.parked + executing bool // guarded by routeMu dispatchesSinceFair uint8 fairSource uint8 } From 59857e8423b5d534bc39d390cb719d7bb1eb7377 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sat, 3 Oct 2026 11:08:55 -0400 Subject: [PATCH 39/50] test(actor): drain the global queue without a lint suppression --- system/scheduler/actor/wake_reuse_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/system/scheduler/actor/wake_reuse_test.go b/system/scheduler/actor/wake_reuse_test.go index 9ff7f5b8d..77d9bc135 100644 --- a/system/scheduler/actor/wake_reuse_test.go +++ b/system/scheduler/actor/wake_reuse_test.go @@ -38,7 +38,10 @@ func TestStaleWakeDoesNotTransitionReusedSlot(t *testing.T) { if got := ProcessState(proc.state.Load()) & stateMask; got != StateBlocked { t.Fatalf("round %d: stale wake moved the reused slot to %s", i, StateName(got)) } - for s.global.Pop() != nil { //nolint:revive // drains the queue + for { + if s.global.Pop() == nil { + break + } } } } From 2559ecfa0059502a4caeee6128882173f2680ae4 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sat, 3 Oct 2026 11:09:23 -0400 Subject: [PATCH 40/50] chore(deps): go-lua with quality-pass fixes --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index bb33f19b4..9ba8e8802 100644 --- a/go.mod +++ b/go.mod @@ -69,7 +69,7 @@ require ( github.com/tree-sitter/tree-sitter-php v0.25.0 github.com/tree-sitter/tree-sitter-python v0.25.0 github.com/tree-sitter/tree-sitter-typescript v0.23.2 - github.com/wippyai/go-lua v1.6.4-0.20261003135019-e6c9f2e920e8 + github.com/wippyai/go-lua v1.6.4-0.20261003145456-1a12b5df98d5 github.com/wippyai/module-registry-proto-go v0.0.2-0.20260908140534-f6e2910c835f github.com/wippyai/tree-sitter-markdown v0.0.3 github.com/wippyai/tree-sitter-sql v0.0.4 diff --git a/go.sum b/go.sum index ec807ebe9..e198b37a0 100644 --- a/go.sum +++ b/go.sum @@ -554,6 +554,8 @@ github.com/wippyai/go-lua v1.6.4-0.20261003025238-2c373075d09d h1:NpoQ7a1P/B1u/h github.com/wippyai/go-lua v1.6.4-0.20261003025238-2c373075d09d/go.mod h1:cwD+390gJgx9cs+Zamby/lch5csqPV9RN1aDEfx/27M= github.com/wippyai/go-lua v1.6.4-0.20261003135019-e6c9f2e920e8 h1:cPF3sBY4gbJi5wDrecxyjkdjtgn4qAFAtnuPsbyO/Kw= github.com/wippyai/go-lua v1.6.4-0.20261003135019-e6c9f2e920e8/go.mod h1:cwD+390gJgx9cs+Zamby/lch5csqPV9RN1aDEfx/27M= +github.com/wippyai/go-lua v1.6.4-0.20261003145456-1a12b5df98d5 h1:sJpAq4jGwV5q/co5qPnfi981X9dPkDKh8bdPGf068Aw= +github.com/wippyai/go-lua v1.6.4-0.20261003145456-1a12b5df98d5/go.mod h1:cwD+390gJgx9cs+Zamby/lch5csqPV9RN1aDEfx/27M= github.com/wippyai/module-registry-proto-go v0.0.2-0.20260908140534-f6e2910c835f h1:AoKS058roqnsHXaJ8Gww7ODz14qJqOK5QBeoALvcq6I= github.com/wippyai/module-registry-proto-go v0.0.2-0.20260908140534-f6e2910c835f/go.mod h1:p4ihYQKRQuqRVLxaH1YL0IEnkz4zKdyfVADOwvA5Tno= github.com/wippyai/tree-sitter-markdown v0.0.3 h1:u6e53+hzPSS848Xhm+I48SwtvWQHrC21wDrogdwCwuc= From a1a72f37b335a023afad0ea97ed9d1412421db6a Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sat, 3 Oct 2026 11:17:46 -0400 Subject: [PATCH 41/50] test(actor): drain the global queue in the loop condition --- system/scheduler/actor/wake_reuse_test.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/system/scheduler/actor/wake_reuse_test.go b/system/scheduler/actor/wake_reuse_test.go index 77d9bc135..9df70b9c0 100644 --- a/system/scheduler/actor/wake_reuse_test.go +++ b/system/scheduler/actor/wake_reuse_test.go @@ -38,10 +38,8 @@ func TestStaleWakeDoesNotTransitionReusedSlot(t *testing.T) { if got := ProcessState(proc.state.Load()) & stateMask; got != StateBlocked { t.Fatalf("round %d: stale wake moved the reused slot to %s", i, StateName(got)) } - for { - if s.global.Pop() == nil { - break - } + for s.global.Pop() != nil { + continue } } } From 0ce0f681dea83568eb13356026ab3e747ef1aa58 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sat, 3 Oct 2026 12:12:01 -0400 Subject: [PATCH 42/50] fix(actor): wake pooled processors on cancellation and signal drain on release CreateProcessor did not register the context-cancel wake that Submit registers, so Terminate or a cancelled context left an idle pooled processor parked forever. ReleaseProcessor lowered the live count without telling a draining Stop, so Stop waited out its whole deadline after the last pooled processor was released. --- .../scheduler/actor/pooled_lifecycle_test.go | 103 ++++++++++++++++++ system/scheduler/actor/scheduler.go | 44 +++++--- 2 files changed, 131 insertions(+), 16 deletions(-) create mode 100644 system/scheduler/actor/pooled_lifecycle_test.go diff --git a/system/scheduler/actor/pooled_lifecycle_test.go b/system/scheduler/actor/pooled_lifecycle_test.go new file mode 100644 index 000000000..c33e1e3e9 --- /dev/null +++ b/system/scheduler/actor/pooled_lifecycle_test.go @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: MPL-2.0 + +package actor + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/wippyai/runtime/api/payload" + pidapi "github.com/wippyai/runtime/api/pid" + "github.com/wippyai/runtime/api/process" + "github.com/wippyai/runtime/api/relay" + sysprocess "github.com/wippyai/runtime/system/process" +) + +// idleForeverProcess waits for messages and never finishes on its own. +type idleForeverProcess struct{} + +func (idleForeverProcess) Init(context.Context, string, payload.Payloads) error { return nil } +func (idleForeverProcess) Step(_ []process.Event, out *process.StepOutput) error { + out.Idle() + return nil +} +func (idleForeverProcess) Send(*relay.Package) error { return nil } +func (idleForeverProcess) Close() {} + +func waitProcessorState(t *testing.T, proc *Processor, want ProcessState) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for ProcessState(proc.state.Load())&stateMask != want { + if time.Now().After(deadline) { + t.Fatalf("processor did not reach %s, is %s", StateName(want), StateName(ProcessState(proc.state.Load()))) + } + time.Sleep(time.Millisecond) + } +} + +// Terminate ends a pooled execution that waits for messages: the owner +// scheduled it, so its result must arrive. +func TestTerminateEndsIdlePooledProcessor(t *testing.T) { + sched := newTestSchedulerWithLifecycle(2, &testLifecycle{}) + sched.Start() + defer testStopScheduler(sched) + + id := pidapi.PID{UniqID: "pooled-idle"} + proc, err := sched.CreateProcessor(context.Background(), id, idleForeverProcess{}) + if err != nil { + t.Fatal(err) + } + results := proc.resultCh + sched.global.Push(proc) + sched.wakeAny() + waitProcessorState(t, proc, StateIdle) + + if err := sched.Terminate(id); err != nil { + t.Fatal(err) + } + select { + case res := <-results: + if !errors.Is(res.Error, sysprocess.ErrTerminated) { + t.Fatalf("expected termination, got %v", res.Error) + } + case <-time.After(5 * time.Second): + t.Fatal("terminated pooled execution never completed") + } + sched.ReleaseProcessor(proc) +} + +// Stop returns as soon as the last pooled processor is released. +func TestStopReturnsWhenLastPooledProcessorIsReleased(t *testing.T) { + sched := newTestSchedulerWithLifecycle(2, &testLifecycle{}) + sched.Start() + + proc, err := sched.CreateProcessor(context.Background(), pidapi.PID{UniqID: "pooled-held"}, idleForeverProcess{}) + if err != nil { + t.Fatal(err) + } + + stopped := make(chan struct{}) + go func() { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + sched.Stop(ctx) + close(stopped) + }() + deadline := time.Now().Add(5 * time.Second) + for !sched.isStopping() { + if time.Now().After(deadline) { + t.Fatal("stop did not begin") + } + time.Sleep(time.Millisecond) + } + time.Sleep(50 * time.Millisecond) + sched.ReleaseProcessor(proc) + + select { + case <-stopped: + case <-time.After(5 * time.Second): + t.Fatal("Stop kept waiting after the last processor was released") + } +} diff --git a/system/scheduler/actor/scheduler.go b/system/scheduler/actor/scheduler.go index bf55544d4..8de05f925 100644 --- a/system/scheduler/actor/scheduler.go +++ b/system/scheduler/actor/scheduler.go @@ -399,15 +399,7 @@ func (s *Scheduler) Submit(ctx context.Context, pid pid.PID, p process.Process, } } - // Cancellation must also wake an actor parked without incoming messages. - // Capture only queue identity/generation: Processor objects are pooled. - q, gen := proc.queue, proc.gen.Load() - stopWake := context.AfterFunc(procCtx, func() { - if q.Push(process.Event{Type: process.EventMessage}, gen) { - s.WakeProcessor(q, gen) - } - }) - proc.cancel = func() { stopWake(); cancel() } + s.wakeOnCancel(proc, procCtx, cancel) // A pooled processor can still be referenced by a stale queue entry from // its previous incarnation; Ready is published only once it is initialized. @@ -418,6 +410,30 @@ func (s *Scheduler) Submit(ctx context.Context, pid pid.PID, p process.Process, return proc, nil } +// wakeOnCancel makes cancellation of the incarnation's context wake an actor +// parked without incoming messages. It captures only queue identity and +// generation, because Processor objects are pooled. +func (s *Scheduler) wakeOnCancel(proc *Processor, ctx context.Context, cancel context.CancelFunc) { + q, gen := proc.queue, proc.gen.Load() + stopWake := context.AfterFunc(ctx, func() { + if q.Push(process.Event{Type: process.EventMessage}, gen) { + s.WakeProcessor(q, gen) + } + }) + proc.cancel = func() { stopWake(); cancel() } +} + +// uncount removes a processor from the live count and tells a draining Stop +// when none remain. +func (s *Scheduler) uncount() { + if s.processorCount.Add(-1) == 0 && s.isStopping() { + select { + case s.drainCh <- struct{}{}: + default: + } + } +} + // Terminate forcibly terminates a process by PID. // Cancels the process context - worker will detect and evict on next step. func (s *Scheduler) Terminate(pid pid.PID) error { @@ -456,12 +472,7 @@ func (s *Scheduler) finishProcessor(proc *Processor, result *process.StepOutput, stopping := s.isStopping() if !proc.pooled { - if s.processorCount.Add(-1) == 0 && stopping { - select { - case s.drainCh <- struct{}{}: - default: - } - } + s.uncount() } if proc.resultCh != nil { @@ -523,6 +534,7 @@ func (s *Scheduler) CreateProcessor(ctx context.Context, pid pid.PID, p process. s.processorCount.Add(1) s.byPID.Store(pid.String(), proc) s.byQueue.Store(proc.queue, proc) + s.wakeOnCancel(proc, procCtx, cancel) // Ready is published only once the processor is initialized; see Submit. proc.state.Store(int32(StateReady)) @@ -530,7 +542,6 @@ func (s *Scheduler) CreateProcessor(ctx context.Context, pid pid.PID, p process. } func (s *Scheduler) ReleaseProcessor(proc *Processor) { - s.processorCount.Add(-1) proc.sig.Store(nil) proc.inspector.Store(nil) s.byPID.Delete(proc.pid.String()) @@ -541,6 +552,7 @@ func (s *Scheduler) ReleaseProcessor(proc *Processor) { if proc.Process != nil { proc.Process.Close() } + s.uncount() } // Send implements relay.Receiver. Routes package to target process. From ba3e3f63c8ffe30c98610d80f6d9b7fa9c4b838a Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sat, 3 Oct 2026 12:31:02 -0400 Subject: [PATCH 43/50] fix(api): align process entry options with v2 Reject unknown meta.options fields on process entries (accepting the v2 process keys and default_host) and parse tick_budget and max_steps with the v2 numeric carriers and ranges, so max_steps spans uint64. Add fuzz targets for options parsing. --- api/runtime/lua/options.go | 118 +++++++- api/runtime/lua/options_fuzz_test.go | 388 +++++++++++++++++++++++++++ api/runtime/lua/options_test.go | 33 +++ 3 files changed, 526 insertions(+), 13 deletions(-) create mode 100644 api/runtime/lua/options_fuzz_test.go diff --git a/api/runtime/lua/options.go b/api/runtime/lua/options.go index 30552c91d..f98ac8190 100644 --- a/api/runtime/lua/options.go +++ b/api/runtime/lua/options.go @@ -6,6 +6,8 @@ import ( "encoding/json" "fmt" "math" + "sort" + "strconv" "github.com/wippyai/runtime/api/attrs" ) @@ -20,6 +22,32 @@ const ( // entryOptionsMetaKey is the entry meta bag holding declaration-time options. const entryOptionsMetaKey = "options" +// Process entry option keys other than the execution limits. They are +// accepted in meta.options so declarations written for the v2 runtime load +// unchanged; only the execution limits are read here. +const ( + ProcessOptionMailboxCapacity = "mailbox_capacity" + ProcessOptionMemoryLimitBytes = "memory_limit_bytes" + ProcessOptionHeapReserveBytes = "heap_reserve_bytes" + ProcessOptionHotPolicyID = "hot_policy_id" + ProcessOptionNetwork = "network" + // ProcessOptionDefaultHost names the host that serves the entry as a + // function; the process function listener reads it from meta.options. + ProcessOptionDefaultHost = "default_host" +) + +// IsProcessEntryOptionKey reports whether key is accepted in the meta.options +// of a process entry. +func IsProcessEntryOptionKey(key string) bool { + switch key { + case ProcessOptionTickBudget, ProcessOptionMaxSteps, ProcessOptionMailboxCapacity, + ProcessOptionMemoryLimitBytes, ProcessOptionHeapReserveBytes, ProcessOptionHotPolicyID, + ProcessOptionNetwork, ProcessOptionDefaultHost: + return true + } + return false +} + // ExecutionBudgets holds the execution limits of a Lua actor. // // TickBudget is the number of VM ticks (backward jumps, loop iterations and @@ -63,11 +91,11 @@ func ExecutionBudgetsFromOptions(options attrs.Attributes, subject string) (Exec b.TickBudget, b.TickBudgetSet = v, true } if raw, ok := options.Get(ProcessOptionMaxSteps); ok && raw != nil { - v, err := optionInt64(raw) - if err != nil || v < 0 { + v, err := optionUint64(raw) + if err != nil { return b, fmt.Errorf("%s %q must be a non-negative integer", subject, ProcessOptionMaxSteps) } - b.MaxSteps, b.MaxStepsSet = uint64(v), true + b.MaxSteps, b.MaxStepsSet = v, true } return b, nil } @@ -87,9 +115,23 @@ func EntryExecutionBudgets(meta attrs.Bag) (ExecutionBudgets, error) { } return ExecutionBudgets{}, nil } + for _, key := range sortedKeys(options) { + if !IsProcessEntryOptionKey(key) { + return ExecutionBudgets{}, fmt.Errorf("process entry meta.%s has unknown field %q", entryOptionsMetaKey, key) + } + } return ExecutionBudgetsFromOptions(options, "process entry meta.options") } +func sortedKeys(options attrs.Bag) []string { + keys := make([]string, 0, len(options)) + for key := range options { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} + // maxExactFloatInteger is the largest integer a float64 holds exactly. const maxExactFloatInteger = 1 << 53 @@ -106,7 +148,7 @@ func optionInt64(raw any) (int64, error) { case int64: return v, nil case uint: - return uintOption(uint64(v)) + return int64FromUint64(uint64(v)) case uint8: return int64(v), nil case uint16: @@ -114,27 +156,77 @@ func optionInt64(raw any) (int64, error) { case uint32: return int64(v), nil case uint64: - return uintOption(v) + return int64FromUint64(v) + case uintptr: + return int64FromUint64(uint64(v)) case float32: - return floatOption(float64(v)) + return int64FromFloat(float64(v)) case float64: - return floatOption(v) + return int64FromFloat(v) case json.Number: return v.Int64() } - return 0, fmt.Errorf("not an integer: %T", raw) + return 0, fmt.Errorf("unsupported %T", raw) } -func uintOption(v uint64) (int64, error) { +func optionUint64(raw any) (uint64, error) { + switch v := raw.(type) { + case uint: + return uint64(v), nil + case uint8: + return uint64(v), nil + case uint16: + return uint64(v), nil + case uint32: + return uint64(v), nil + case uint64: + return v, nil + case uintptr: + return uint64(v), nil + case int: + return nonNegative(uint64(v), v < 0) + case int8: + return nonNegative(uint64(v), v < 0) + case int16: + return nonNegative(uint64(v), v < 0) + case int32: + return nonNegative(uint64(v), v < 0) + case int64: + return nonNegative(uint64(v), v < 0) + case float32: + return uint64FromFloat(float64(v)) + case float64: + return uint64FromFloat(v) + case json.Number: + return strconv.ParseUint(v.String(), 10, 64) + } + return 0, fmt.Errorf("unsupported %T", raw) +} + +func nonNegative(v uint64, negative bool) (uint64, error) { + if negative { + return 0, fmt.Errorf("negative") + } + return v, nil +} + +func int64FromUint64(v uint64) (int64, error) { if v > math.MaxInt64 { - return 0, fmt.Errorf("integer overflows int64: %d", v) + return 0, fmt.Errorf("overflow") } return int64(v), nil } -func floatOption(v float64) (int64, error) { - if v != math.Trunc(v) || math.Abs(v) > maxExactFloatInteger { - return 0, fmt.Errorf("not an exact integer: %v", v) +func int64FromFloat(v float64) (int64, error) { + if math.IsNaN(v) || math.IsInf(v, 0) || v < -maxExactFloatInteger || v > maxExactFloatInteger || math.Trunc(v) != v { + return 0, fmt.Errorf("invalid float") } return int64(v), nil } + +func uint64FromFloat(v float64) (uint64, error) { + if math.IsNaN(v) || math.IsInf(v, 0) || v < 0 || v > maxExactFloatInteger || math.Trunc(v) != v { + return 0, fmt.Errorf("invalid float") + } + return uint64(v), nil +} diff --git a/api/runtime/lua/options_fuzz_test.go b/api/runtime/lua/options_fuzz_test.go new file mode 100644 index 000000000..49e2da918 --- /dev/null +++ b/api/runtime/lua/options_fuzz_test.go @@ -0,0 +1,388 @@ +// SPDX-License-Identifier: MPL-2.0 + +package lua + +import ( + "encoding/json" + "fmt" + "math" + "strconv" + "testing" + + "github.com/wippyai/runtime/api/attrs" +) + +// optionKinds is the number of Go representations fuzzOptionValue produces. +const optionKinds = 21 + +// fuzzOptionValue builds an option value of the representation selected by +// kind from the fuzzed scalars. +func fuzzOptionValue(kind uint8, i int64, u uint64, f float64, s string) any { + switch kind % optionKinds { + case 0: + return int(i) + case 1: + return int8(i) + case 2: + return int16(i) + case 3: + return int32(i) + case 4: + return i + case 5: + return uint(u) + case 6: + return uint8(u) + case 7: + return uint16(u) + case 8: + return uint32(u) + case 9: + return u + case 10: + return float32(f) + case 11: + return f + case 12: + return json.Number(s) + case 13: + return s + case 14: + return i%2 == 0 + case 15: + return nil + case 16: + return []any{i} + case 17: + return map[string]any{"v": i} + case 18: + return []byte(s) + case 19: + return uintptr(u) + default: + return struct{}{} + } +} + +// wantTick is the reference reading of a tick_budget value: whether it is +// given, whether it is a valid integer, and its value. +func wantTick(raw any) (given, valid bool, v int64) { + switch x := raw.(type) { + case nil: + return false, false, 0 + case int: + return true, true, int64(x) + case int8: + return true, true, int64(x) + case int16: + return true, true, int64(x) + case int32: + return true, true, int64(x) + case int64: + return true, true, x + case uint: + return true, uint64(x) <= math.MaxInt64, int64(x) + case uint8: + return true, true, int64(x) + case uint16: + return true, true, int64(x) + case uint32: + return true, true, int64(x) + case uint64: + return true, x <= math.MaxInt64, int64(x) + case uintptr: + return true, uint64(x) <= math.MaxInt64, int64(x) + case float32: + return wantFloat(float64(x), true) + case float64: + return wantFloat(x, true) + case json.Number: + n, err := strconv.ParseInt(string(x), 10, 64) + return true, err == nil, n + } + return true, false, 0 +} + +// wantSteps is the reference reading of a max_steps value, which spans the +// whole uint64 range and rejects negatives. +func wantSteps(raw any) (given, valid bool, v uint64) { + switch x := raw.(type) { + case nil: + return false, false, 0 + case int: + return true, x >= 0, uint64(x) + case int8: + return true, x >= 0, uint64(x) + case int16: + return true, x >= 0, uint64(x) + case int32: + return true, x >= 0, uint64(x) + case int64: + return true, x >= 0, uint64(x) + case uint: + return true, true, uint64(x) + case uint8: + return true, true, uint64(x) + case uint16: + return true, true, uint64(x) + case uint32: + return true, true, uint64(x) + case uint64: + return true, true, x + case uintptr: + return true, true, uint64(x) + case float32: + g, ok, n := wantFloat(float64(x), false) + return g, ok, uint64(n) + case float64: + g, ok, n := wantFloat(x, false) + return g, ok, uint64(n) + case json.Number: + n, err := strconv.ParseUint(string(x), 10, 64) + return true, err == nil, n + } + return true, false, 0 +} + +func wantFloat(f float64, signed bool) (given, valid bool, v int64) { + if math.IsNaN(f) || math.IsInf(f, 0) || f != math.Trunc(f) || f > 1<<53 || f < -(1<<53) || (!signed && f < 0) { + return true, false, 0 + } + return true, true, int64(f) +} + +func checkTickBudget(t *testing.T, b ExecutionBudgets, err error, raw any) { + t.Helper() + given, valid, want := wantTick(raw) + switch { + case !given: + if err != nil || b.TickBudgetSet || b.TickBudget != 0 { + t.Fatalf("tick_budget %#v: absent option parsed to %+v, %v", raw, b, err) + } + case !valid: + if err == nil { + t.Fatalf("tick_budget %#v: expected an error, got %+v", raw, b) + } + default: + if err != nil || !b.TickBudgetSet || b.TickBudget != want { + t.Fatalf("tick_budget %#v: got %+v, %v, want %d", raw, b, err, want) + } + } +} + +func checkMaxSteps(t *testing.T, b ExecutionBudgets, err error, raw any) { + t.Helper() + given, valid, want := wantSteps(raw) + switch { + case !given: + if err != nil || b.MaxStepsSet || b.MaxSteps != 0 { + t.Fatalf("max_steps %#v: absent option parsed to %+v, %v", raw, b, err) + } + case !valid: + if err == nil { + t.Fatalf("max_steps %#v: expected an error, got %+v", raw, b) + } + default: + if err != nil || !b.MaxStepsSet || b.MaxSteps != want { + t.Fatalf("max_steps %#v: got %+v, %v, want %d", raw, b, err, want) + } + } +} + +func addOptionSeeds(f *testing.F) { + f.Helper() + ints := []int64{0, 1, -1, 2048, math.MaxInt64, math.MinInt64, 1 << 53, 1<<53 + 1} + f.Add(uint8(9), int64(0), uint64(math.MaxUint64), 0.0, "", uint8(9), int64(0), uint64(math.MaxUint64), 0.0, "18446744073709551615") + for kind := uint8(0); kind < optionKinds; kind++ { + for _, i := range ints { + f.Add(kind, i, uint64(i), float64(i), "100", kind+1, i, uint64(i), float64(i), "7") + } + f.Add(kind, int64(5), uint64(math.MaxUint64), math.NaN(), "1.5", kind, int64(-5), uint64(1)<<63, math.Inf(1), "1e3") + f.Add(kind, int64(5), uint64(5), 0.5, "", kind, int64(5), uint64(5), -0.0, "99999999999999999999") + } +} + +// ExecutionBudgetsFromOptions reads tick_budget and max_steps with the +// documented semantics for every value representation. +func FuzzExecutionBudgetsFromOptions(f *testing.F) { + addOptionSeeds(f) + f.Fuzz(func(t *testing.T, tk uint8, ti int64, tu uint64, tf float64, ts string, mk uint8, mi int64, mu uint64, mf float64, ms string) { + tick := fuzzOptionValue(tk, ti, tu, tf, ts) + steps := fuzzOptionValue(mk, mi, mu, mf, ms) + + b, err := ExecutionBudgetsFromOptions(attrs.Bag{ProcessOptionTickBudget: tick}, "fuzz") + checkTickBudget(t, b, err, tick) + if b.MaxStepsSet { + t.Fatalf("max_steps reported set by tick_budget only: %+v", b) + } + + b, err = ExecutionBudgetsFromOptions(attrs.Bag{ProcessOptionMaxSteps: steps}, "fuzz") + checkMaxSteps(t, b, err, steps) + if b.TickBudgetSet { + t.Fatalf("tick_budget reported set by max_steps only: %+v", b) + } + + both, err := ExecutionBudgetsFromOptions(attrs.Bag{ProcessOptionTickBudget: tick, ProcessOptionMaxSteps: steps}, "fuzz") + tg, tv, _ := wantTick(tick) + sg, sv, _ := wantSteps(steps) + wantErr := (tg && !tv) || (sg && !sv) + if wantErr != (err != nil) { + t.Fatalf("both %#v %#v: error %v, want error %v", tick, steps, err, wantErr) + } + if err == nil { + single, _ := ExecutionBudgetsFromOptions(attrs.Bag{ProcessOptionTickBudget: tick}, "fuzz") + other, _ := ExecutionBudgetsFromOptions(attrs.Bag{ProcessOptionMaxSteps: steps}, "fuzz") + if both.TickBudget != single.TickBudget || both.TickBudgetSet != single.TickBudgetSet || + both.MaxSteps != other.MaxSteps || both.MaxStepsSet != other.MaxStepsSet { + t.Fatalf("options are not independent: both %+v tick %+v steps %+v", both, single, other) + } + } + }) +} + +// wrapOptions places the options under meta.options in one of the map +// representations an entry meta can hold. +func wrapOptions(shape uint8, options attrs.Bag) any { + switch shape % 3 { + case 0: + return map[string]any(options) + case 1: + return options + default: + return attrs.Attributes(options) + } +} + +// EntryExecutionBudgets reads only meta.options and rejects the options +// placed directly in meta or in a non-map meta.options. +func FuzzEntryExecutionBudgets(f *testing.F) { + f.Add(uint8(0), uint8(4), int64(2048), uint64(0), 0.0, "", uint8(4), int64(10), uint64(0), 0.0, "", uint8(0), uint8(0)) + f.Add(uint8(1), uint8(11), int64(0), uint64(0), 1.5, "", uint8(13), int64(0), uint64(0), 0.0, "x", uint8(1), uint8(1)) + f.Add(uint8(2), uint8(4), int64(-1), uint64(0), 0.0, "", uint8(4), int64(-1), uint64(0), 0.0, "", uint8(2), uint8(2)) + f.Fuzz(func(t *testing.T, shape uint8, tk uint8, ti int64, tu uint64, tf float64, ts string, mk uint8, mi int64, mu uint64, mf float64, ms string, direct uint8, outer uint8) { + tick := fuzzOptionValue(tk, ti, tu, tf, ts) + steps := fuzzOptionValue(mk, mi, mu, mf, ms) + options := attrs.Bag{ProcessOptionTickBudget: tick, ProcessOptionMaxSteps: steps} + + meta := attrs.Bag{entryOptionsMetaKey: wrapOptions(shape, options), "comment": "x"} + switch direct % 3 { + case 1: + meta[ProcessOptionTickBudget] = ti + case 2: + meta[ProcessOptionMaxSteps] = mi + } + got, err := EntryExecutionBudgets(meta) + if direct%3 != 0 { + if err == nil { + t.Fatalf("option placed directly in meta accepted: %+v", got) + } + return + } + want, wantErr := ExecutionBudgetsFromOptions(options, "entry") + if (err != nil) != (wantErr != nil) { + t.Fatalf("meta.options %#v: error %v, direct parse error %v", options, err, wantErr) + } + if err == nil && got != want { + t.Fatalf("meta.options %#v: got %+v, direct parse %+v", options, got, want) + } + + scalars := []any{tick, steps, "fast", int64(5), true, []any{"tick_budget"}, 1.5} + raw := scalars[int(outer)%len(scalars)] + _, err = EntryExecutionBudgets(attrs.Bag{entryOptionsMetaKey: raw}) + switch raw.(type) { + case map[string]any, attrs.Bag: + if err != nil { + t.Fatalf("map meta.options rejected: %v", err) + } + case nil: + if err != nil { + t.Fatalf("null meta.options rejected: %v", err) + } + default: + if err == nil { + t.Fatalf("meta.options %T accepted", raw) + } + } + }) +} + +// Parsing an entry and a spawn bag then overriding follows the precedence +// rule: an option the spawn gives replaces the entry's, even an explicit zero, +// and an option it omits leaves the entry's value. +func FuzzExecutionBudgetsOverride(f *testing.F) { + f.Add(int64(100), int64(5), uint8(0), int64(-1), int64(0), uint8(1), uint8(1)) + f.Add(int64(0), int64(0), uint8(2), int64(7), int64(7), uint8(0), uint8(2)) + f.Add(int64(-1), int64(1<<40), uint8(3), int64(0), int64(0), uint8(3), uint8(3)) + f.Fuzz(func(t *testing.T, et, es int64, em uint8, st, ss int64, sm uint8, _ uint8) { + entryOpts := attrs.Bag{} + spawnOpts := attrs.Bag{} + if em&1 != 0 { + entryOpts[ProcessOptionTickBudget] = et + } + if em&2 != 0 { + entryOpts[ProcessOptionMaxSteps] = es + } + if sm&1 != 0 { + spawnOpts[ProcessOptionTickBudget] = st + } + if sm&2 != 0 { + spawnOpts[ProcessOptionMaxSteps] = ss + } + entry, entryErr := EntryExecutionBudgets(attrs.Bag{entryOptionsMetaKey: entryOpts}) + spawn, spawnErr := ExecutionBudgetsFromOptions(spawnOpts, "spawn") + if entryErr != nil || spawnErr != nil { + // Only max_steps below zero is invalid among int64 inputs. + if (entryErr != nil) != (em&2 != 0 && es < 0) || (spawnErr != nil) != (sm&2 != 0 && ss < 0) { + t.Fatalf("unexpected errors: entry %v spawn %v", entryErr, spawnErr) + } + return + } + got := entry.Override(spawn) + + wantTick, wantTickSet := entry.TickBudget, entry.TickBudgetSet + if sm&1 != 0 { + wantTick, wantTickSet = st, true + } + wantSteps, wantStepsSet := entry.MaxSteps, entry.MaxStepsSet + if sm&2 != 0 { + wantSteps, wantStepsSet = uint64(ss), true + } + if got.TickBudget != wantTick || got.TickBudgetSet != wantTickSet || got.MaxSteps != wantSteps || got.MaxStepsSet != wantStepsSet { + t.Fatalf("override(%+v, %+v) = %+v", entry, spawn, got) + } + if (ExecutionBudgets{}).Override(spawn) != spawn { + t.Fatalf("zero budgets overridden by %+v changed it", spawn) + } + if entry.Override(ExecutionBudgets{}) != entry { + t.Fatalf("empty override changed %+v", entry) + } + }) +} + +// A process entry accepts the v2 process option keys and default_host in +// meta.options, and rejects every other key by name. +func FuzzEntryOptionKeys(f *testing.F) { + for _, k := range []string{"max_step", "tick_budgets", "default_host", "network", "hot_policy_id", "mailbox_capacity", "", "TICK_BUDGET", "pool"} { + f.Add(k, int64(3)) + } + known := map[string]bool{ + "mailbox_capacity": true, "memory_limit_bytes": true, "heap_reserve_bytes": true, + "hot_policy_id": true, "network": true, "default_host": true, + } + f.Fuzz(func(t *testing.T, key string, v int64) { + if key == ProcessOptionTickBudget || key == ProcessOptionMaxSteps { + t.Skip() + } + _, err := EntryExecutionBudgets(attrs.Bag{entryOptionsMetaKey: attrs.Bag{key: v}}) + if known[key] { + if err != nil { + t.Fatalf("known option %q rejected: %v", key, err) + } + return + } + want := fmt.Sprintf("process entry meta.options has unknown field %q", key) + if err == nil || err.Error() != want { + t.Fatalf("unknown option %q: got %v, want %q", key, err, want) + } + }) +} diff --git a/api/runtime/lua/options_test.go b/api/runtime/lua/options_test.go index 9a4dafc49..c1b292ce3 100644 --- a/api/runtime/lua/options_test.go +++ b/api/runtime/lua/options_test.go @@ -88,3 +88,36 @@ func TestExecutionBudgetsFromNilOptionsIsEmpty(t *testing.T) { t.Fatalf("expected empty budgets, got %+v %v", b, err) } } + +func TestEntryExecutionBudgetsRejectsUnknownOptionFields(t *testing.T) { + _, err := EntryExecutionBudgets(attrs.Bag{"options": attrs.Bag{"max_step": 5}}) + if err == nil || err.Error() != `process entry meta.options has unknown field "max_step"` { + t.Fatalf("unexpected error %v", err) + } +} + +func TestEntryExecutionBudgetsAcceptsProcessEntryOptionKeys(t *testing.T) { + options := attrs.Bag{} + for _, key := range []string{"mailbox_capacity", "memory_limit_bytes", "heap_reserve_bytes", "hot_policy_id", "network", "default_host"} { + options[key] = "x" + } + if _, err := EntryExecutionBudgets(attrs.Bag{"options": options}); err != nil { + t.Fatal(err) + } +} + +func TestExecutionBudgetsMaxStepsSpansUint64(t *testing.T) { + const max = ^uint64(0) + for _, raw := range []any{max, uint(max), uintptr(max), json.Number("18446744073709551615")} { + b, err := ExecutionBudgetsFromOptions(attrs.Bag{ProcessOptionMaxSteps: raw}, "test") + if err != nil || !b.MaxStepsSet || b.MaxSteps != max { + t.Fatalf("max_steps %#v: got %+v %v", raw, b, err) + } + } + if _, err := ExecutionBudgetsFromOptions(attrs.Bag{ProcessOptionTickBudget: uint64(1) << 63}, "test"); err == nil { + t.Fatal("tick_budget above int64 must be rejected") + } + if b, err := ExecutionBudgetsFromOptions(attrs.Bag{ProcessOptionTickBudget: uintptr(9)}, "test"); err != nil || b.TickBudget != 9 { + t.Fatalf("uintptr tick_budget: %+v %v", b, err) + } +} From 0a697bda50a101b21823c5ece7d5fdfe8212fee7 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sat, 3 Oct 2026 12:31:14 -0400 Subject: [PATCH 44/50] docs(engine): describe when chunk environments seal import aliases --- runtime/lua/engine/process_factory.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/runtime/lua/engine/process_factory.go b/runtime/lua/engine/process_factory.go index df838cc81..ee9e4b491 100644 --- a/runtime/lua/engine/process_factory.go +++ b/runtime/lua/engine/process_factory.go @@ -299,10 +299,13 @@ func (f *ProcessFactory) isolationBinder( // through a scoped require) and falls back to the shared safe base for standard // globals via its metatable. // -// A runtime-installed DSL global may temporarily shadow an import alias when -// that global did not exist in the base environment at chunk creation time. This -// preserves the old shared-_G migration DSL behavior without leaking declared -// imports into _G. Undeclared modules resolve to nil and require fails closed. +// An alias is an import when the base environment holds a value under that +// name at the moment the environment is sealed: after library initialization +// completes, or at the first lookup if that comes earlier. A global installed +// under an alias name before sealing is shadowed by the import; one installed +// afterwards, such as the migration DSL's helpers, stays visible through the +// base without leaking declared imports into _G. Undeclared modules resolve to +// nil and require fails closed. func buildChunkEnv(l *lua.LState, base *lua.LTable, imports []code.Import, valueByNode map[registry.ID]lua.LValue) (*lua.LTable, func()) { env := l.NewTable() From e9d639ba4922082198cc795cea31fec75b80ba6c Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sat, 3 Oct 2026 12:31:14 -0400 Subject: [PATCH 45/50] test: seeded chaos tests for the actor scheduler and Lua actors --- runtime/lua/engine/chaos_test.go | 729 ++++++++++++++++++++ system/scheduler/actor/chaos_test.go | 964 +++++++++++++++++++++++++++ 2 files changed, 1693 insertions(+) create mode 100644 runtime/lua/engine/chaos_test.go create mode 100644 system/scheduler/actor/chaos_test.go diff --git a/runtime/lua/engine/chaos_test.go b/runtime/lua/engine/chaos_test.go new file mode 100644 index 000000000..b4b71b305 --- /dev/null +++ b/runtime/lua/engine/chaos_test.go @@ -0,0 +1,729 @@ +// SPDX-License-Identifier: MPL-2.0 + +package engine + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "math" + "math/rand" + "os" + goruntime "runtime" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + lua "github.com/wippyai/go-lua" + "github.com/wippyai/runtime/api/attrs" + ctxapi "github.com/wippyai/runtime/api/context" + "github.com/wippyai/runtime/api/dispatcher" + "github.com/wippyai/runtime/api/payload" + pidapi "github.com/wippyai/runtime/api/pid" + "github.com/wippyai/runtime/api/process" + "github.com/wippyai/runtime/api/registry" + "github.com/wippyai/runtime/api/relay" + "github.com/wippyai/runtime/api/runtime" + luaapi "github.com/wippyai/runtime/api/runtime/lua" + scheduler "github.com/wippyai/runtime/system/scheduler/actor" +) + +// Engine chaos: real Lua actors under the actor scheduler with random +// tick_budget and max_steps, driven by a seed. Environment: +// +// WIPPY_LUA_CHAOS_SEED first seed (default 1) +// WIPPY_LUA_CHAOS_RUNS number of consecutive seeds (default 3, 100 with WIPPY_CHAOS_LONG) +// WIPPY_LUA_CHAOS_PROCS actors per run (default 40, 150 with WIPPY_CHAOS_LONG) +func luaChaosEnvInt(name string, short, long int) int { + def := short + if os.Getenv("WIPPY_CHAOS_LONG") != "" { + def = long + } + if v := os.Getenv(name); v != "" { + if n, err := strconv.Atoi(v); err == nil { + return n + } + } + return def +} + +var errLuaChaosStart = errors.New("lua chaos: start rejected") + +type luaStage struct { + body string // Lua statements that leave the stage result in `result` + expected string // value of `result` + // entry budgets of the incarnation + entry luaapi.ExecutionBudgets + // delayed completion left outstanding by the incarnation before it upgrades + staleMs int + // upgrade requests an in-place upgrade after the body + upgrade bool +} + +type luaRec struct { + rng *rand.Rand + pid pidapi.PID + cancel context.CancelFunc + spawn attrs.Bag + spawnBudget luaapi.ExecutionBudgets + ready chan struct{} + stages []luaStage + result *runtime.Result + id int + completes atomic.Int32 + starts atomic.Int32 + upgrades atomic.Int32 + submitted atomic.Bool + gaveUp atomic.Bool + killed atomic.Bool + running atomic.Int32 + blocker bool + failing bool + wantsMsgs bool + wantsWake bool + resultMu sync.Mutex +} + +// limited reports whether any incarnation of the actor can be stopped by a +// max_steps limit. +func (r *luaRec) limited() bool { + if r.spawnBudget.MaxSteps > 0 { + return true + } + if r.spawnBudget.MaxStepsSet { + return false + } + for _, s := range r.stages { + if s.entry.MaxSteps > 0 { + return true + } + } + return false +} + +// maxStepsOf is the effective max_steps of an incarnation. +func (r *luaRec) maxStepsOf(stage luaStage) uint64 { + return stage.entry.Override(r.spawnBudget).MaxSteps +} + +type luaChaos struct { + sched *scheduler.Scheduler + appCtx context.Context + recs []*luaRec + incs []*chaosLua + violations []string + seed int64 + mu sync.Mutex + stopped atomic.Bool +} + +func (w *luaChaos) violate(format string, args ...any) { + w.mu.Lock() + defer w.mu.Unlock() + if len(w.violations) < 40 { + w.violations = append(w.violations, fmt.Sprintf(format, args...)) + } +} + +// chaosLua observes one incarnation's Lua process. +type chaosLua struct { + *Process + w *luaChaos + rec *luaRec + inc int + limit uint64 + inits atomic.Int32 + closed atomic.Int32 + running atomic.Int32 + limited atomic.Bool +} + +func (p *chaosLua) Init(ctx context.Context, method string, input payload.Payloads) error { + p.inits.Add(1) + return p.Process.Init(ctx, method, input) +} + +func (p *chaosLua) Step(events []process.Event, out *process.StepOutput) error { + if !p.running.CompareAndSwap(0, 1) { + p.w.violate("rec %d inc %d: concurrent steps of one incarnation", p.rec.id, p.inc) + } + defer p.running.Store(0) + if p.rec.running.Add(1) != 1 { + p.w.violate("rec %d: steps of two incarnations overlap", p.rec.id) + } + defer p.rec.running.Add(-1) + if p.rec.completes.Load() != 0 { + p.w.violate("rec %d inc %d: step after completion", p.rec.id, p.inc) + } + if p.closed.Load() != 0 { + p.w.violate("rec %d inc %d: step after close", p.rec.id, p.inc) + } + if p.limited.Load() { + p.w.violate("rec %d inc %d: step after the step limit failed it", p.rec.id, p.inc) + } + + err := p.Process.Step(events, out) + switch { + case errors.Is(err, process.ErrStepLimitExceeded): + p.limited.Store(true) + if p.limit == 0 { + p.w.violate("rec %d inc %d: step limit error without max_steps", p.rec.id, p.inc) + } + case p.limit > 0 && p.Process.StepsUsed() > p.limit: + p.w.violate("rec %d inc %d: %d steps used with max_steps %d (err %v)", p.rec.id, p.inc, p.Process.StepsUsed(), p.limit, err) + } + return err +} + +func (p *chaosLua) Close() { + if p.closed.Add(1) != 1 { + p.w.violate("rec %d inc %d: closed more than once", p.rec.id, p.inc) + return + } + if p.running.Load() != 0 { + p.w.violate("rec %d inc %d: closed during a step", p.rec.id, p.inc) + } + p.Process.Close() +} + +type luaChaosFactory struct{ w *luaChaos } + +func (f luaChaosFactory) Create(id registry.ID) (process.Process, *process.Meta, error) { + n, err := strconv.Atoi(id.Name) + if err != nil || n < 0 || n >= len(f.w.recs) { + return nil, nil, fmt.Errorf("unknown chaos source %v", id) + } + rec := f.w.recs[n] + inc := int(rec.upgrades.Add(1)) + if inc >= len(rec.stages) { + f.w.violate("rec %d: upgrade to incarnation %d but there are %d", rec.id, inc, len(rec.stages)) + return nil, nil, errLuaChaosStart + } + p, err := f.w.newProc(rec, inc) + if err != nil { + return nil, nil, err + } + return p, &process.Meta{}, nil +} + +func (w *luaChaos) newProc(rec *luaRec, inc int) (*chaosLua, error) { + stage := rec.stages[inc] + src := w.script(rec, inc) + upgradeReq := &UpgradeRequest{Source: registry.NewID("chaos", strconv.Itoa(rec.id))} + proc, err := NewProcess( + WithScript(src, fmt.Sprintf("chaos-%d-%d.lua", rec.id, inc)), + WithProcessExecutionBudgets(stage.entry), + WithModuleBinder(LoadCoreModules), + WithModuleBinder(bindTestYield), + WithModuleBinder(wrapBinder(func(l *lua.LState) { l.SetGlobal("upgrade_request", upgradeReq) })), + ) + if err != nil { + return nil, err + } + p := &chaosLua{Process: proc, w: w, rec: rec, inc: inc, limit: rec.maxStepsOf(stage)} + w.mu.Lock() + w.incs = append(w.incs, p) + w.mu.Unlock() + return p, nil +} + +func (w *luaChaos) script(rec *luaRec, inc int) string { + stage := rec.stages[inc] + var b strings.Builder + b.WriteString(stage.body) + b.WriteString("\n") + if stage.upgrade { + if stage.staleMs > 0 { + // A message wakes the actor while its spawned coroutine's yield is + // still outstanding; the actor then upgrades. + fmt.Fprintf(&b, "local wake = channel.new(1)\nsubscribe('wake', wake)\ncoroutine.spawn(function() test_yield(%d) end)\ncoroutine.yield()\nwake:receive()\n", stage.staleMs) + } + b.WriteString("coroutine.yield(upgrade_request)\n") + } + fmt.Fprintf(&b, "return \"gen%d:\" .. tostring(result)\n", inc) + return b.String() +} + +// luaChaosHandler completes a test_yield with its duration in milliseconds, +// after that duration: an actor that receives another value was resumed with +// a completion of a different yield. +func luaChaosHandler() dispatcher.Handler { + return dispatcher.HandlerFunc(func(_ context.Context, cmd dispatcher.Command, tag uint64, r dispatcher.ResultReceiver) error { + d := cmd.(testYieldCmd).Duration + ms := d.Milliseconds() + if d == 0 { + r.CompleteYield(tag, ms, nil) + return nil + } + time.AfterFunc(d, func() { r.CompleteYield(tag, ms, nil) }) + return nil + }) +} + +func luaTriangle(n int) int { return n * (n + 1) / 2 } + +// luaStageBody builds the body of a stage and its expected result. +func luaStageBody(rng *rand.Rand, rec *luaRec) (body, expected string) { + switch rng.Intn(6) { + case 0: + n := 100 + rng.Intn(4000) + return fmt.Sprintf("local s = 0\nfor i = 1, %d do s = s + i end\nlocal result = s", n), strconv.Itoa(luaTriangle(n)) + case 1: + k, n, y := 2+rng.Intn(5), 50+rng.Intn(400), 1+rng.Intn(40) + return fmt.Sprintf(` +local ch = channel.new(%[1]d) +for c = 1, %[1]d do + coroutine.spawn(function() + local s = 0 + for i = 1, %[2]d do + s = s + i + if i %% %[3]d == 0 then coroutine.yield() end + end + ch:send(s) + end) +end +local total = 0 +for c = 1, %[1]d do total = total + ch:receive() end +local result = total`, k, n, y), strconv.Itoa(k * luaTriangle(n)) + case 2: + r := 1 + rng.Intn(4) + var sb strings.Builder + sum := 0 + sb.WriteString("local acc = 0\n") + for i := 0; i < r; i++ { + ms := rng.Intn(5) + fmt.Fprintf(&sb, "do local r = test_yield(%d) if tonumber(r) ~= %d then error('wrong completion ' .. tostring(r) .. ' for %d') end acc = acc + tonumber(r) end\n", ms, ms, ms) + sum += ms + } + sb.WriteString("local result = acc") + return sb.String(), strconv.Itoa(sum) + case 3: + k := 2 + rng.Intn(4) + var sum int + for c := 1; c <= k; c++ { + sum += c + } + return fmt.Sprintf(` +local ch = channel.new(%[1]d) +for c = 1, %[1]d do + coroutine.spawn(function() + local spin = 0 + for i = 1, 200 do spin = spin + i end + local r = test_yield(c) + if tonumber(r) ~= c then error('wrong completion ' .. tostring(r) .. ' for ' .. c) end + ch:send(c) + end) +end +local total = 0 +for c = 1, %[1]d do total = total + ch:receive() end +local result = total`, k), strconv.Itoa(sum) + case 4: + m := 3 + rng.Intn(40) + rec.wantsMsgs = true + return fmt.Sprintf(` +local ch = channel.new(64) +subscribe("msg", ch) +local n = 0 +while n < %d do + ch:receive() + n = n + 1 + for i = 1, 50 do n = n + 0 end +end +local result = n`, m), strconv.Itoa(m) + default: + n := 20 + rng.Intn(500) + return fmt.Sprintf(` +local function fib(n) if n < 2 then return n end return fib(n - 1) + fib(n - 2) end +local result = fib(%d)`, 5+n%12), strconv.Itoa(fibRef(5 + n%12)) + } +} + +func fibRef(n int) int { + if n < 2 { + return n + } + return fibRef(n-1) + fibRef(n-2) +} + +func luaBudgetPick(rng *rand.Rand) luaapi.ExecutionBudgets { + var b luaapi.ExecutionBudgets + if rng.Intn(3) != 0 { + ticks := []int64{0, -1, 1, 2, 7, 64, 1000, math.MaxInt64} + b.TickBudget, b.TickBudgetSet = ticks[rng.Intn(len(ticks))], true + } + if rng.Intn(4) == 0 { + steps := []uint64{0, 3, 12, 40, 1_000_000} + b.MaxSteps, b.MaxStepsSet = steps[rng.Intn(len(steps))], true + } + return b +} + +// optionValue renders an integer option in one of the representations spawn +// options arrive in. +func optionValue(rng *rand.Rand, v int64) any { + switch rng.Intn(4) { + case 0: + return v + case 1: + return int(v) + case 2: + return float64(v) + default: + return json.Number(strconv.FormatInt(v, 10)) + } +} + +func luaChaosPlan(rng *rand.Rand, rec *luaRec) { + kind := rng.Intn(10) + incarnations := 1 + if kind <= 3 { + incarnations = 2 + rng.Intn(3) + } + for i := 0; i < incarnations; i++ { + var st luaStage + last := i == incarnations-1 + switch { + case last && kind == 4: + rec.failing = true + st.body = "local s = 0 for i = 1, 300 do s = s + i end\nerror('chaos-boom')\nlocal result = s" + case last && kind == 5: + rec.blocker = true + st.body = "local ch = channel.new(1)\nsubscribe('never', ch)\nch:receive()\nlocal result = 0" + case last && kind == 6: + rec.blocker = true + st.body = "local n = 0\nwhile true do n = n + 1 end\nlocal result = n" + default: + st.body, st.expected = luaStageBody(rng, rec) + } + st.entry = luaBudgetPick(rng) + if !last { + st.upgrade = true + if rng.Intn(3) == 0 { + // The replacement waits on a yield longer than the stale one. + st.staleMs = 8 + rng.Intn(8) + rec.wantsWake = true + } + } + rec.stages = append(rec.stages, st) + } + for i, st := range rec.stages { + if st.staleMs > 0 && i+1 < len(rec.stages) { + nxt := &rec.stages[i+1] + nxt.body = "do local r = test_yield(40) if tonumber(r) ~= 40 then error('wrong completion ' .. tostring(r) .. ' for 40') end end\n" + nxt.body + } + } + if rng.Intn(3) == 0 { + var spawn luaapi.ExecutionBudgets + bag := attrs.Bag{} + if rng.Intn(2) == 0 { + v := []int64{0, -1, 1, 5, 100, 5000}[rng.Intn(6)] + bag[luaapi.ProcessOptionTickBudget] = optionValue(rng, v) + spawn.TickBudget, spawn.TickBudgetSet = v, true + } + if rng.Intn(2) == 0 { + v := []int64{0, 4, 15, 60, 2_000_000}[rng.Intn(5)] + bag[luaapi.ProcessOptionMaxSteps] = optionValue(rng, v) + spawn.MaxSteps, spawn.MaxStepsSet = uint64(v), true + } + rec.spawn, rec.spawnBudget = bag, spawn + } +} + +type luaChaosLifecycle struct{ w *luaChaos } + +func (l luaChaosLifecycle) recFor(p pidapi.PID) *luaRec { + n, err := strconv.Atoi(strings.TrimPrefix(p.UniqID, "luachaos-")) + if err != nil || n < 0 || n >= len(l.w.recs) { + return nil + } + return l.w.recs[n] +} + +func (l luaChaosLifecycle) OnStart(_ context.Context, p pidapi.PID, _ process.Process) error { + if rec := l.recFor(p); rec != nil && rec.starts.Add(1) != 1 { + l.w.violate("rec %d: started more than once", rec.id) + } + return nil +} + +func (l luaChaosLifecycle) OnComplete(_ context.Context, p pidapi.PID, res *runtime.Result) { + rec := l.recFor(p) + if rec == nil { + return + } + if rec.running.Load() != 0 { + l.w.violate("rec %d: completed during a step", rec.id) + } + if rec.completes.Add(1) != 1 { + l.w.violate("rec %d: completed more than once", rec.id) + return + } + rec.resultMu.Lock() + rec.result = res + rec.resultMu.Unlock() +} + +func (w *luaChaos) submit(rec *luaRec) { + proc, err := w.newProc(rec, 0) + if err != nil { + w.violate("rec %d: cannot build the process: %v", rec.id, err) + rec.gaveUp.Store(true) + close(rec.ready) + return + } + frame, fc := ctxapi.OpenFrameContext(w.appCtx) + if err := runtime.SetFrameID(frame, registry.NewID("chaos", strconv.Itoa(rec.id))); err != nil { + w.violate("rec %d: %v", rec.id, err) + } + if rec.spawn != nil { + if err := fc.Set(runtime.FrameLifecycleOptionsKey, attrs.Attributes(rec.spawn)); err != nil { + w.violate("rec %d: %v", rec.id, err) + } + } + ctx, cancel := context.WithCancel(frame) + rec.cancel = cancel + _, err = w.sched.Submit(ctx, rec.pid, proc, "", nil) + if err != nil { + cancel() + proc.Close() + if !errors.Is(err, process.ErrSchedulerStopping) { + w.violate("rec %d: unexpected Submit error %v", rec.id, err) + } + rec.gaveUp.Store(true) + close(rec.ready) + return + } + rec.submitted.Store(true) + close(rec.ready) +} + +func TestLuaChaos(t *testing.T) { + seed := int64(luaChaosEnvInt("WIPPY_LUA_CHAOS_SEED", 1, 1)) + runs := luaChaosEnvInt("WIPPY_LUA_CHAOS_RUNS", 3, 100) + for i := 0; i < runs; i++ { + s := seed + int64(i) + t.Run(fmt.Sprintf("seed=%d", s), func(t *testing.T) { runLuaChaos(t, s) }) + } +} + +func runLuaChaos(t *testing.T, seed int64) { + procs := luaChaosEnvInt("WIPPY_LUA_CHAOS_PROCS", 40, 150) + rng := rand.New(rand.NewSource(seed)) + workers := 1 + rng.Intn(4) + stopMode := rng.Intn(3) + t.Logf("lua chaos seed=%d workers=%d procs=%d stopMode=%d (replay: WIPPY_LUA_CHAOS_SEED=%d WIPPY_LUA_CHAOS_RUNS=1)", seed, workers, procs, stopMode, seed) + + baseline := goruntime.NumGoroutine() + w := &luaChaos{seed: seed} + reg := newMockRegistry() + reg.Register(testYieldCmdID, luaChaosHandler()) + w.sched = scheduler.NewScheduler(reg, scheduler.WithWorkers(workers), scheduler.WithLifecycle(luaChaosLifecycle{w: w})) + + appCtx := ctxapi.WithAppContext(context.Background(), ctxapi.NewAppContext()) + process.WithFactory(appCtx, luaChaosFactory{w: w}) + w.appCtx = appCtx + + for i := 0; i < procs; i++ { + rec := &luaRec{id: i, pid: pidapi.PID{Host: "test", UniqID: fmt.Sprintf("luachaos-%d", i)}, ready: make(chan struct{})} + rec.rng = rand.New(rand.NewSource(seed*1000003 + int64(i))) + luaChaosPlan(rec.rng, rec) + w.recs = append(w.recs, rec) + } + + w.sched.Start() + + var wg sync.WaitGroup + for _, rec := range w.recs { + wg.Add(1) + go func() { + defer wg.Done() + time.Sleep(time.Duration(rec.rng.Intn(30_000)) * time.Microsecond) + w.submit(rec) + }() + wg.Add(1) + go func() { + defer wg.Done() + <-rec.ready + if !rec.submitted.Load() { + return + } + r := rec.rng + deadline := time.Now().Add(20 * time.Second) + killAt := time.Now().Add(time.Duration(r.Intn(40)) * time.Millisecond) + kill := rec.blocker || r.Intn(8) == 0 + for rec.completes.Load() == 0 && time.Now().Before(deadline) { + if rec.wantsMsgs { + pkg := relay.NewPackage(pidapi.PID{}, rec.pid, "msg", payload.NewPayload(lua.LString("x"), payload.Lua)) + if err := w.sched.Send(pkg); err != nil && !errors.Is(err, process.ErrProcessNotFound) && !errors.Is(err, process.ErrProcessClosed) { + w.violate("rec %d: unexpected Send error %v", rec.id, err) + } + } + if rec.wantsWake { + pkg := relay.NewPackage(pidapi.PID{}, rec.pid, "wake", payload.NewPayload(lua.LString("x"), payload.Lua)) + _ = w.sched.Send(pkg) + } + if kill && time.Now().After(killAt) { + rec.killed.Store(true) + if r.Intn(2) == 0 { + rec.cancel() + } else if err := w.sched.Terminate(rec.pid); err != nil && !errors.Is(err, process.ErrProcessNotFound) { + w.violate("rec %d: unexpected Terminate error %v", rec.id, err) + } + return + } + time.Sleep(time.Duration(100+r.Intn(900)) * time.Microsecond) + } + }() + } + + stopTimeout := 20 * time.Second + if stopMode == 2 { + stopTimeout = time.Duration(20+rng.Intn(80)) * time.Millisecond + } + stopDone := make(chan struct{}) + startStop := func() { + w.stopped.Store(true) + ctx, cancel := context.WithTimeout(context.Background(), stopTimeout) + w.sched.Stop(ctx) + cancel() + close(stopDone) + } + if stopMode != 0 { + time.AfterFunc(time.Duration(rng.Intn(80))*time.Millisecond, func() { go startStop() }) + } + + wg.Wait() + if stopMode == 0 { + w.awaitSettled(t, 15*time.Second) + go startStop() + } + select { + case <-stopDone: + case <-time.After(60 * time.Second): + t.Fatalf("seed %d: Stop did not return", seed) + } + w.awaitSettled(t, 10*time.Second) + w.verify(t, seed) + + deadline := time.Now().Add(10 * time.Second) + for goruntime.NumGoroutine() > baseline+2 && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + if n := goruntime.NumGoroutine(); n > baseline+2 { + buf := make([]byte, 1<<20) + t.Errorf("seed %d: %d goroutines remain, baseline %d\n%s", seed, n, baseline, buf[:goruntime.Stack(buf, true)]) + } +} + +func (w *luaChaos) awaitSettled(t *testing.T, d time.Duration) { + t.Helper() + deadline := time.Now().Add(d) + for { + var stranded []string + for _, rec := range w.recs { + select { + case <-rec.ready: + default: + stranded = append(stranded, fmt.Sprintf("rec %d never submitted", rec.id)) + continue + } + if rec.submitted.Load() && rec.completes.Load() == 0 { + stranded = append(stranded, fmt.Sprintf("rec %d (blocker=%v stages=%d) stranded", rec.id, rec.blocker, len(rec.stages))) + } + } + if len(stranded) == 0 { + return + } + if time.Now().After(deadline) { + t.Errorf("seed %d: %d actors stranded: %v\nprocesses: %+v", w.seed, len(stranded), stranded[:min(len(stranded), 10)], w.sched.ListProcesses()) + return + } + time.Sleep(5 * time.Millisecond) + } +} + +func (w *luaChaos) verify(t *testing.T, seed int64) { + t.Helper() + outcomes := map[string]int{} + for _, rec := range w.recs { + if rec.gaveUp.Load() { + outcomes["rejected"]++ + if rec.completes.Load() != 0 { + w.violate("rec %d: completed although admission failed", rec.id) + } + continue + } + if n := rec.completes.Load(); n != 1 { + w.violate("rec %d: completed %d times", rec.id, n) + continue + } + rec.resultMu.Lock() + res := rec.result + rec.resultMu.Unlock() + outcomes[w.checkOutcome(rec, res)]++ + } + t.Logf("seed %d outcomes %v", seed, outcomes) + + w.mu.Lock() + incs := append([]*chaosLua(nil), w.incs...) + w.mu.Unlock() + for _, p := range incs { + if p.inits.Load() > 0 && p.closed.Load() != 1 { + w.violate("rec %d inc %d: closed %d times, want once", p.rec.id, p.inc, p.closed.Load()) + } + } + + stats := w.sched.Stats() + if stats["processes"] != 0 { + w.violate("%d processes still registered after stop", stats["processes"]) + } + if stats["global_queue"] != 0 { + w.violate("global queue holds %d entries after stop", stats["global_queue"]) + } + if len(w.violations) > 0 { + t.Errorf("seed %d: %d invariant violations (replay: WIPPY_LUA_CHAOS_SEED=%d WIPPY_LUA_CHAOS_RUNS=1):\n %s", + seed, len(w.violations), seed, strings.Join(w.violations, "\n ")) + } +} + +// checkOutcome verifies a finished actor and names its outcome class. +func (w *luaChaos) checkOutcome(rec *luaRec, res *runtime.Result) string { + if res == nil { + w.violate("rec %d: no result", rec.id) + return "none" + } + killed := rec.killed.Load() || w.stopped.Load() + last := rec.stages[len(rec.stages)-1] + if res.Error == nil { + var got string + if res.Value != nil { + if v, ok := res.Value.Data().(lua.LValue); ok { + got = lua.LVAsString(v) + } + } + want := fmt.Sprintf("gen%d:%s", len(rec.stages)-1, last.expected) + if rec.failing || rec.blocker { + w.violate("rec %d: finished with %q although its final stage cannot finish", rec.id, got) + } else if got != want { + w.violate("rec %d: result %q, want %q", rec.id, got, want) + } + return "done" + } + switch { + case errors.Is(res.Error, process.ErrStepLimitExceeded): + if !rec.limited() { + w.violate("rec %d: step limit error without max_steps", rec.id) + } + return "limit" + case rec.failing && strings.Contains(res.Error.Error(), "chaos-boom"): + return "fail" + case killed: + return "killed" + } + w.violate("rec %d: unexpected error %v", rec.id, res.Error) + return "unexpected" +} diff --git a/system/scheduler/actor/chaos_test.go b/system/scheduler/actor/chaos_test.go new file mode 100644 index 000000000..f1f95daff --- /dev/null +++ b/system/scheduler/actor/chaos_test.go @@ -0,0 +1,964 @@ +// SPDX-License-Identifier: MPL-2.0 + +package actor + +import ( + "context" + "errors" + "fmt" + "math/rand" + "os" + goruntime "runtime" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + ctxapi "github.com/wippyai/runtime/api/context" + "github.com/wippyai/runtime/api/dispatcher" + "github.com/wippyai/runtime/api/payload" + pidapi "github.com/wippyai/runtime/api/pid" + "github.com/wippyai/runtime/api/process" + "github.com/wippyai/runtime/api/registry" + "github.com/wippyai/runtime/api/relay" + apiruntime "github.com/wippyai/runtime/api/runtime" + "github.com/wippyai/runtime/api/topology" + sysprocess "github.com/wippyai/runtime/system/process" + "github.com/wippyai/runtime/system/scheduler" +) + +// Chaos configuration. Defaults keep a normal run fast; the environment +// selects longer runs and replays a failing seed: +// +// WIPPY_CHAOS_SEED first seed (default 1) +// WIPPY_CHAOS_RUNS number of consecutive seeds (default 4, 200 with WIPPY_CHAOS_LONG) +// WIPPY_CHAOS_PROCS processes per run (default 120, 600 with WIPPY_CHAOS_LONG) +// WIPPY_CHAOS_LONG any non-empty value selects the long defaults +func chaosEnvInt(name string, short, long int) int { + def := short + if os.Getenv("WIPPY_CHAOS_LONG") != "" { + def = long + } + if v := os.Getenv(name); v != "" { + if n, err := strconv.Atoi(v); err == nil { + return n + } + } + return def +} + +const cmdChaosAsync dispatcher.CommandID = 100 + +// cmdChaosUnregistered has no handler, so a yield of it completes with an +// unknown-command error. +const cmdChaosUnregistered dispatcher.CommandID = 101 + +var ( + errChaosFail = errors.New("chaos: planned failure") + errChaosHandler = errors.New("chaos: handler failure") + errChaosCancel = errors.New("chaos: cancelled") + errChaosStart = errors.New("chaos: start rejected") +) + +type chaosAsyncMode uint8 + +const ( + asyncLater chaosAsyncMode = iota // completes from another goroutine after a delay + asyncInline // completes inside Handle + asyncHandleErr // Handle returns an error + asyncLaterError // completes with an error after a delay + asyncUnknown // no handler +) + +type chaosAsyncCmd struct { + token chaosToken + delay time.Duration + mode chaosAsyncMode +} + +func (c chaosAsyncCmd) CmdID() dispatcher.CommandID { + if c.mode == asyncUnknown { + return cmdChaosUnregistered + } + return cmdChaosAsync +} + +// chaosToken identifies the yield a completion answers. A completion that +// reaches an incarnation other than the one that issued it is a defect. +type chaosToken struct { + rec int + inc int + tag uint64 +} + +type chaosOpKind uint8 + +const ( + opSpin chaosOpKind = iota // n steps, each reporting Preempt or Continue + opAsync // yield and wait for the completion + opYieldPreempt // yield and report Preempt in the same step, then wait + opWaitMsg // wait until n messages arrived in total + opStaleUpgrade // yield with a long delay, preempt, upgrade without waiting + opUpgrade + opFail + opFinish + opNever // wait for termination +) + +type chaosOp struct { + delay time.Duration + n int + mode chaosAsyncMode + kind chaosOpKind +} + +// chaosRec is the shared state of one process across its incarnations. +type chaosRec struct { + result *apiruntime.Result + rng *rand.Rand + plans [][]chaosOp + pid pidapi.PID + cancelParent context.CancelFunc + ready chan struct{} + maxSteps uint64 + need int64 + id int + stepsRun atomic.Uint64 + msgs atomic.Int64 + completes atomic.Int32 + starts atomic.Int32 + running atomic.Int32 + upgrades atomic.Int32 + killed atomic.Bool + submitted atomic.Bool + gaveUp atomic.Bool + failUpgrade int + failStart bool + deaf bool + blocker bool + pooled bool + failUpInit bool + resultMu sync.Mutex +} + +func (r *chaosRec) expectsLimit() bool { return r.maxSteps > 0 } + +type chaosWorld struct { + sched *Scheduler + appCtx context.Context + t *testing.T + recs []*chaosRec + incs []*chaosProc + violations []string + seed int64 + mu sync.Mutex + stopped atomic.Bool +} + +func (w *chaosWorld) violate(format string, args ...any) { + w.mu.Lock() + defer w.mu.Unlock() + if len(w.violations) < 40 { + w.violations = append(w.violations, fmt.Sprintf(format, args...)) + } +} + +func (w *chaosWorld) track(p *chaosProc) { + w.mu.Lock() + w.incs = append(w.incs, p) + w.mu.Unlock() +} + +// chaosProc is one incarnation of a chaos actor. +type chaosProc struct { + w *chaosWorld + rec *chaosRec + plan []chaosOp + rng *rand.Rand + outstanding map[uint64]struct{} + inits atomic.Int32 + closed atomic.Int32 + running atomic.Int32 + steps uint64 + nextTag uint64 + waitTag uint64 + pc int + ph int + inc int + preempt bool + limited bool +} + +var ( + _ process.Process = (*chaosProc)(nil) + _ process.Preemptible = (*chaosProc)(nil) + _ process.StepAccounted = (*chaosProc)(nil) +) + +func (p *chaosProc) EnablePreemption() { p.preempt = true } +func (p *chaosProc) StepsUsed() uint64 { return p.steps } +func (p *chaosProc) ResumeStepCount(used uint64) { p.steps = used } + +func (p *chaosProc) Init(context.Context, string, payload.Payloads) error { + if p.inits.Add(1) != 1 { + p.w.violate("rec %d inc %d: Init called more than once", p.rec.id, p.inc) + } + if p.rec.failUpInit && p.inc > 0 { + return errChaosStart + } + return nil +} + +func (p *chaosProc) Close() { + if p.closed.Add(1) != 1 { + p.w.violate("rec %d inc %d: closed more than once", p.rec.id, p.inc) + } + if p.running.Load() != 0 { + p.w.violate("rec %d inc %d: closed during a step", p.rec.id, p.inc) + } +} + +func isCancelEvent(data any) bool { + pkg, ok := data.(*relay.Package) + if !ok || pkg == nil || len(pkg.Messages) != 1 || pkg.Messages[0].Topic != topology.TopicEvents || len(pkg.Messages[0].Payloads) != 1 { + return false + } + _, ok = pkg.Messages[0].Payloads[0].Data().(*topology.CancelEvent) + return ok +} + +func (p *chaosProc) Step(events []process.Event, out *process.StepOutput) error { + rec := p.rec + if !p.running.CompareAndSwap(0, 1) { + p.w.violate("rec %d inc %d: concurrent steps of one incarnation", rec.id, p.inc) + } + defer p.running.Store(0) + if rec.running.Add(1) != 1 { + p.w.violate("rec %d: steps of two incarnations overlap", rec.id) + } + defer rec.running.Add(-1) + + if rec.completes.Load() != 0 { + p.w.violate("rec %d inc %d: step after completion", rec.id, p.inc) + } + if p.closed.Load() != 0 { + p.w.violate("rec %d inc %d: step after close", rec.id, p.inc) + } + if p.limited { + p.w.violate("rec %d inc %d: step after the step limit failed it", rec.id, p.inc) + } + + p.steps++ + if rec.maxSteps > 0 && p.steps > rec.maxSteps { + p.limited = true + return process.ErrStepLimitExceeded + } + total := rec.stepsRun.Add(1) + if rec.maxSteps > 0 && total > rec.maxSteps { + p.w.violate("rec %d: %d steps ran with max_steps %d", rec.id, total, rec.maxSteps) + } + + cancelled := false + for _, e := range events { + switch e.Type { + case process.EventYieldComplete: + p.absorb(e) + case process.EventMessage: + if isCancelEvent(e.Data) { + cancelled = true + } else if e.Data != nil { + rec.msgs.Add(1) + } + } + } + if cancelled && !rec.deaf { + return errChaosCancel + } + return p.advance(out) +} + +func (p *chaosProc) absorb(e process.Event) { + rec := p.rec + if _, ok := p.outstanding[e.Tag]; !ok { + p.w.violate("rec %d inc %d: completion for tag %d that is not outstanding (data %#v)", rec.id, p.inc, e.Tag, e.Data) + return + } + delete(p.outstanding, e.Tag) + if e.Error != nil { + return + } + tok, ok := e.Data.(chaosToken) + if !ok || tok.rec != rec.id || tok.inc != p.inc || tok.tag != e.Tag { + p.w.violate("rec %d inc %d: completion %#v delivered to the wrong yield (tag %d)", rec.id, p.inc, e.Data, e.Tag) + } +} + +func (p *chaosProc) yield(out *process.StepOutput, op chaosOp) uint64 { + p.nextTag++ + tag := p.nextTag + p.outstanding[tag] = struct{}{} + out.Yield(chaosAsyncCmd{token: chaosToken{rec: p.rec.id, inc: p.inc, tag: tag}, delay: op.delay, mode: op.mode}, tag) + return tag +} + +func (p *chaosProc) next() { p.pc++; p.ph = 0 } + +func (p *chaosProc) advance(out *process.StepOutput) error { + for { + if p.pc >= len(p.plan) { + out.Done(payload.New(chaosToken{rec: p.rec.id, inc: p.inc})) + return nil + } + op := p.plan[p.pc] + switch op.kind { + case opSpin: + if p.ph >= op.n { + p.next() + continue + } + p.ph++ + if p.rng.Intn(3) == 0 { + out.Continue() + } else { + out.Preempt() + } + return nil + case opAsync: + if p.ph == 0 { + p.waitTag = p.yield(out, op) + p.ph = 1 + out.WaitForYields() + return nil + } + if _, pending := p.outstanding[p.waitTag]; pending { + out.WaitForYields() + return nil + } + p.next() + case opYieldPreempt: + if p.ph == 0 { + p.waitTag = p.yield(out, op) + p.ph = 1 + out.Preempt() + return nil + } + if _, pending := p.outstanding[p.waitTag]; pending { + out.WaitForYields() + return nil + } + p.next() + case opWaitMsg: + if p.rec.msgs.Load() >= int64(op.n) { + p.next() + continue + } + out.Idle() + return nil + case opStaleUpgrade: + if p.ph == 0 { + p.yield(out, op) + p.ph = 1 + out.Preempt() + return nil + } + out.SetUpgrade(&process.UpgradeRequest{Source: registry.NewID("chaos", strconv.Itoa(p.rec.id))}) + return nil + case opUpgrade: + out.SetUpgrade(&process.UpgradeRequest{Source: registry.NewID("chaos", strconv.Itoa(p.rec.id))}) + return nil + case opFail: + return errChaosFail + case opFinish: + out.Done(payload.New(chaosToken{rec: p.rec.id, inc: p.inc})) + return nil + case opNever: + out.Idle() + return nil + } + } +} + +type chaosFactory struct{ w *chaosWorld } + +func (f chaosFactory) Create(id registry.ID) (process.Process, *process.Meta, error) { + n, err := strconv.Atoi(id.Name) + if err != nil || n < 0 || n >= len(f.w.recs) { + return nil, nil, fmt.Errorf("unknown chaos source %v", id) + } + rec := f.w.recs[n] + inc := int(rec.upgrades.Add(1)) + if rec.failUpgrade == inc { + return nil, nil, errChaosStart + } + if inc >= len(rec.plans) { + f.w.violate("rec %d: upgrade to incarnation %d but the plan has %d", rec.id, inc, len(rec.plans)) + return nil, nil, errChaosStart + } + return f.w.newProc(rec, inc), &process.Meta{Method: "main"}, nil +} + +func (w *chaosWorld) newProc(rec *chaosRec, inc int) *chaosProc { + p := &chaosProc{ + w: w, + rec: rec, + inc: inc, + plan: rec.plans[inc], + rng: rand.New(rand.NewSource(int64(rec.id)*7919 + int64(inc) + w.seed)), + outstanding: make(map[uint64]struct{}, 4), + } + w.track(p) + return p +} + +func chaosHandler() dispatcher.Handler { + return dispatcher.HandlerFunc(func(_ context.Context, cmd dispatcher.Command, tag uint64, r dispatcher.ResultReceiver) error { + c := cmd.(chaosAsyncCmd) + switch c.mode { + case asyncInline: + r.CompleteYield(tag, c.token, nil) + case asyncHandleErr: + return errChaosHandler + case asyncLaterError: + time.AfterFunc(c.delay, func() { r.CompleteYield(tag, nil, errChaosHandler) }) + default: + time.AfterFunc(c.delay, func() { r.CompleteYield(tag, c.token, nil) }) + } + return nil + }) +} + +type chaosLifecycle struct{ w *chaosWorld } + +func (l chaosLifecycle) recFor(p pidapi.PID) *chaosRec { + n, err := strconv.Atoi(strings.TrimPrefix(p.UniqID, "chaos-")) + if err != nil || n < 0 || n >= len(l.w.recs) { + return nil + } + return l.w.recs[n] +} + +func (l chaosLifecycle) OnStart(_ context.Context, p pidapi.PID, _ process.Process) error { + rec := l.recFor(p) + if rec == nil { + return nil + } + if rec.starts.Add(1) != 1 { + l.w.violate("rec %d: started more than once", rec.id) + } + if rec.failStart { + return errChaosStart + } + return nil +} + +func (l chaosLifecycle) OnComplete(_ context.Context, p pidapi.PID, res *apiruntime.Result) { + rec := l.recFor(p) + if rec == nil { + return + } + if rec.running.Load() != 0 { + l.w.violate("rec %d: completed during a step", rec.id) + } + if rec.completes.Add(1) != 1 { + l.w.violate("rec %d: completed more than once", rec.id) + return + } + rec.resultMu.Lock() + rec.result = res + rec.resultMu.Unlock() +} + +func chaosPlanFor(rng *rand.Rand, rec *chaosRec) { + randDelay := func() time.Duration { + switch rng.Intn(10) { + case 0: + return 0 + case 1: + return time.Duration(5+rng.Intn(15)) * time.Millisecond + default: + return time.Duration(rng.Intn(1500)) * time.Microsecond + } + } + middle := func() []chaosOp { + var ops []chaosOp + for i := rng.Intn(5); i > 0; i-- { + switch rng.Intn(6) { + case 0: + ops = append(ops, chaosOp{kind: opSpin, n: 1 + rng.Intn(12)}) + case 1, 2: + ops = append(ops, chaosOp{kind: opAsync, delay: randDelay(), mode: chaosAsyncMode(rng.Intn(5))}) + case 3: + ops = append(ops, chaosOp{kind: opYieldPreempt, delay: randDelay(), mode: asyncLater}) + case 4: + ops = append(ops, chaosOp{kind: opAsync, delay: randDelay(), mode: asyncLaterError}) + default: + ops = append(ops, chaosOp{kind: opSpin, n: 1 + rng.Intn(3)}) + } + } + return ops + } + + kind := rng.Intn(10) + incarnations := 1 + if kind <= 3 { + incarnations = 2 + rng.Intn(3) + } + stale := false + for i := 0; i < incarnations; i++ { + plan := middle() + if stale { + // The replacement waits on a yield with the same tag the replaced + // code left outstanding. + plan = append([]chaosOp{{kind: opAsync, delay: 12 * time.Millisecond, mode: asyncLater}}, plan...) + stale = false + } + if i < incarnations-1 { + if rng.Intn(3) == 0 { + plan = append(plan, chaosOp{kind: opStaleUpgrade, delay: time.Duration(1+rng.Intn(4)) * time.Millisecond}) + stale = true + } else { + plan = append(plan, chaosOp{kind: opUpgrade}) + } + } else { + switch { + case kind == 4: + plan = append(plan, chaosOp{kind: opFail}) + case kind == 5: + rec.blocker = true + plan = append(plan, chaosOp{kind: opNever}) + case kind == 6: + rec.need = int64(50 + rng.Intn(400)) + plan = append(plan, chaosOp{kind: opWaitMsg, n: int(rec.need)}, chaosOp{kind: opFinish}) + default: + plan = append(plan, chaosOp{kind: opFinish}) + } + } + if i == 0 && kind != 6 && rng.Intn(3) == 0 { + n := 1 + rng.Intn(20) + plan = append([]chaosOp{{kind: opWaitMsg, n: n}}, plan...) + if int64(n) > rec.need { + rec.need = int64(n) + } + } + rec.plans = append(rec.plans, plan) + } + if rng.Intn(5) == 0 { + rec.maxSteps = uint64(3 + rng.Intn(60)) + } + rec.deaf = rng.Intn(8) == 0 + if incarnations > 1 { + switch rng.Intn(10) { + case 0: + rec.failUpgrade = 1 + rng.Intn(incarnations-1) + case 1: + rec.failUpInit = true + } + } + rec.failStart = rng.Intn(25) == 0 +} + +func (w *chaosWorld) submit(rec *chaosRec) { + proc := w.newProc(rec, 0) + frame, fc := ctxapi.OpenFrameContext(w.appCtx) + fc.Seal() + ctx, cancel := context.WithCancel(frame) + rec.cancelParent = cancel + _, err := w.sched.Submit(ctx, rec.pid, proc, "main", nil) + if err != nil { + cancel() + proc.Close() + switch { + case errors.Is(err, process.ErrSchedulerStopping), errors.Is(err, process.ErrMaxProcessesExceeded), errors.Is(err, errChaosStart): + default: + w.violate("rec %d: unexpected Submit error %v", rec.id, err) + } + if rec.completes.Load() != 0 { + w.violate("rec %d: completed although Submit failed", rec.id) + } + rec.gaveUp.Store(true) + close(rec.ready) + return + } + rec.submitted.Store(true) + close(rec.ready) +} + +// submitPooled runs a process the way a pooled executor does: the owner +// creates the processor, schedules it, reads its result and releases it. +func (w *chaosWorld) submitPooled(rec *chaosRec) { + proc := w.newProc(rec, 0) + frame, fc := ctxapi.OpenFrameContext(w.appCtx) + fc.Seal() + ctx, cancel := context.WithCancel(frame) + defer cancel() + pr, err := w.sched.CreateProcessor(ctx, rec.pid, proc) + if err != nil { + proc.Close() + if !errors.Is(err, process.ErrSchedulerStopping) && !errors.Is(err, process.ErrMaxProcessesExceeded) { + w.violate("rec %d: unexpected CreateProcessor error %v", rec.id, err) + } + rec.gaveUp.Store(true) + close(rec.ready) + return + } + rec.submitted.Store(true) + rec.cancelParent = cancel + close(rec.ready) + results := pr.resultCh + if err := proc.Init(ctx, "main", nil); err != nil { + w.sched.ReleaseProcessor(pr) + return + } + w.sched.global.Push(pr) + w.sched.wakeAny() + select { + case <-results: + case <-time.After(30 * time.Second): + w.violate("rec %d: pooled execution did not complete", rec.id) + } + w.sched.ReleaseProcessor(pr) +} + +func (w *chaosWorld) allowedSendError(err error) bool { + return err == nil || errors.Is(err, process.ErrProcessNotFound) || errors.Is(err, process.ErrProcessClosed) +} + +func (w *chaosWorld) send(rec *chaosRec) { + err := w.sched.Send(&relay.Package{Target: rec.pid, Messages: []*relay.Message{{Topic: "chaos"}}}) + if !w.allowedSendError(err) { + w.violate("rec %d: unexpected Send error %v", rec.id, err) + } +} + +func chaosGoroutineSettle(baseline int, d time.Duration) (int, bool) { + deadline := time.Now().Add(d) + for { + n := goruntime.NumGoroutine() + if n <= baseline+2 { + return n, true + } + if time.Now().After(deadline) { + return n, false + } + time.Sleep(10 * time.Millisecond) + } +} + +func TestSchedulerChaos(t *testing.T) { + seed := int64(chaosEnvInt("WIPPY_CHAOS_SEED", 1, 1)) + runs := chaosEnvInt("WIPPY_CHAOS_RUNS", 4, 200) + for i := 0; i < runs; i++ { + s := seed + int64(i) + t.Run(fmt.Sprintf("seed=%d", s), func(t *testing.T) { runSchedulerChaos(t, s) }) + } +} + +func runSchedulerChaos(t *testing.T, seed int64) { + procs := chaosEnvInt("WIPPY_CHAOS_PROCS", 120, 600) + rng := rand.New(rand.NewSource(seed)) + workers := 1 + rng.Intn(6) + maxProcs := int64(0) + if rng.Intn(4) == 0 { + maxProcs = int64(procs / 2) + } + stopMode := rng.Intn(3) // 0: stop after the run, 1: stop at a random time, 2: stop at a random time with a short deadline + t.Logf("chaos seed=%d workers=%d procs=%d maxProcs=%d stopMode=%d (replay: WIPPY_CHAOS_SEED=%d WIPPY_CHAOS_RUNS=1)", seed, workers, procs, maxProcs, stopMode, seed) + + baseline := goruntime.NumGoroutine() + + w := &chaosWorld{t: t, seed: seed} + reg := scheduler.NewRegistry() + reg.Register(cmdChaosAsync, chaosHandler()) + opts := []Option{WithWorkers(workers), WithLifecycle(chaosLifecycle{w: w}), WithLocalQueueSize(8 << rng.Intn(5))} + if maxProcs > 0 { + opts = append(opts, WithMaxProcesses(maxProcs)) + } + w.sched = NewScheduler(reg, opts...) + + appCtx := ctxapi.WithAppContext(context.Background(), ctxapi.NewAppContext()) + process.WithFactory(appCtx, chaosFactory{w: w}) + w.appCtx = appCtx + + for i := 0; i < procs; i++ { + rec := &chaosRec{id: i, pid: pidapi.PID{UniqID: fmt.Sprintf("chaos-%d", i)}, ready: make(chan struct{})} + rec.rng = rand.New(rand.NewSource(seed*1000003 + int64(i))) + chaosPlanFor(rec.rng, rec) + rec.pooled = rec.rng.Intn(12) == 0 && len(rec.plans) == 1 + if rec.pooled { + rec.failStart = false + rec.failUpgrade = 0 + } + w.recs = append(w.recs, rec) + } + + w.sched.Start() + + var wg sync.WaitGroup + for _, rec := range w.recs { + rec := rec + wg.Add(1) + go func() { + defer wg.Done() + time.Sleep(time.Duration(rec.rng.Intn(40_000)) * time.Microsecond) + if rec.pooled { + w.submitPooled(rec) + } else { + w.submit(rec) + } + }() + wg.Add(1) + go func() { + defer wg.Done() + <-rec.ready + if !rec.submitted.Load() { + return + } + r := rec.rng + for sent := int64(0); sent < rec.need+int64(r.Intn(8)); sent++ { + w.send(rec) + if r.Intn(16) == 0 { + time.Sleep(time.Duration(r.Intn(500)) * time.Microsecond) + } + } + w.send(&chaosRec{pid: pidapi.PID{UniqID: "chaos-missing"}, id: -1}) + time.Sleep(time.Duration(r.Intn(15)) * time.Millisecond) + if rec.blocker || r.Intn(7) == 0 { + rec.killed.Store(true) + if r.Intn(2) == 0 { + rec.cancelParent() + } else if err := w.sched.Terminate(rec.pid); err != nil && !errors.Is(err, process.ErrProcessNotFound) { + w.violate("rec %d: unexpected Terminate error %v", rec.id, err) + } + } + }() + } + + if rng.Intn(2) == 0 { + resizeSeed := rng.Int63() + wg.Add(1) + go func() { + defer wg.Done() + r := rand.New(rand.NewSource(resizeSeed)) + for i := 0; i < 20; i++ { + time.Sleep(time.Duration(r.Intn(4000)) * time.Microsecond) + if err := w.sched.ResizeWorkers(1 + r.Intn(6)); err != nil && !errors.Is(err, process.ErrSchedulerStopping) { + w.violate("unexpected ResizeWorkers error %v", err) + } + } + }() + } + + stopTimeout := 20 * time.Second + if stopMode == 2 { + stopTimeout = time.Duration(20+rng.Intn(80)) * time.Millisecond + } + stopDone := make(chan struct{}) + startStop := func() { + w.stopped.Store(true) + started := time.Now() + ctx, cancel := context.WithTimeout(context.Background(), stopTimeout) + w.sched.Stop(ctx) + cancel() + if stopMode != 2 && time.Since(started) > stopTimeout-time.Second { + w.violate("Stop took %v, close to its %v deadline", time.Since(started), stopTimeout) + } + close(stopDone) + } + if stopMode != 0 { + time.AfterFunc(time.Duration(rng.Intn(60))*time.Millisecond, func() { go startStop() }) + } + + wg.Wait() + if stopMode == 0 { + w.awaitSettled(t, 30*time.Second) + go startStop() + } + select { + case <-stopDone: + case <-time.After(60 * time.Second): + t.Fatalf("seed %d: Stop did not return (goroutines dumped below)\n%s", seed, chaosStacks()) + } + w.awaitSettled(t, 10*time.Second) + + w.verify(t, seed) + + if n, ok := chaosGoroutineSettle(baseline, 10*time.Second); !ok { + t.Errorf("seed %d: %d goroutines remain, baseline %d\n%s", seed, n, baseline, chaosStacks()) + } +} + +func chaosStacks() string { + buf := make([]byte, 1<<20) + return string(buf[:goruntime.Stack(buf, true)]) +} + +// awaitSettled waits until every record reached a terminal state. +func (w *chaosWorld) awaitSettled(t *testing.T, d time.Duration) { + t.Helper() + deadline := time.Now().Add(d) + for { + var stranded []string + for _, rec := range w.recs { + select { + case <-rec.ready: + default: + stranded = append(stranded, fmt.Sprintf("rec %d never submitted", rec.id)) + continue + } + if rec.submitted.Load() && rec.completes.Load() == 0 { + stranded = append(stranded, fmt.Sprintf("rec %d (pooled=%v blocker=%v deaf=%v plans=%d) stranded", rec.id, rec.pooled, rec.blocker, rec.deaf, len(rec.plans))) + } + } + if len(stranded) == 0 { + return + } + if time.Now().After(deadline) { + t.Errorf("seed %d: %d processes stranded: %v\nprocesses: %+v", w.seed, len(stranded), stranded[:min(len(stranded), 10)], w.sched.ListProcesses()) + return + } + time.Sleep(5 * time.Millisecond) + } +} + +func (w *chaosWorld) verify(t *testing.T, seed int64) { + t.Helper() + outcomes := map[string]int{} + upgrades, rejected := 0, 0 + for _, rec := range w.recs { + starts, completes := rec.starts.Load(), rec.completes.Load() + if rec.gaveUp.Load() { + rejected++ + if completes != 0 { + w.violate("rec %d: completed although admission failed", rec.id) + } + continue + } + if completes != 1 { + w.violate("rec %d: completed %d times (starts %d)", rec.id, completes, starts) + continue + } + rec.resultMu.Lock() + res := rec.result + rec.resultMu.Unlock() + w.checkOutcome(rec, res) + outcomes[outcomeName(res)]++ + upgrades += int(rec.upgrades.Load()) + } + t.Logf("seed %d outcomes %v admitted-failures=%d upgrades=%d", seed, outcomes, rejected, upgrades) + + w.mu.Lock() + incs := append([]*chaosProc(nil), w.incs...) + w.mu.Unlock() + for _, p := range incs { + if p.inits.Load() > 0 && p.closed.Load() != 1 { + w.violate("rec %d inc %d: closed %d times, want once", p.rec.id, p.inc, p.closed.Load()) + } + } + + s := w.sched + if n := s.parked.Load(); n != 0 { + w.violate("parked counter = %d after stop", n) + } + for _, wk := range s.workerSnapshot() { + if wk.parked.Load() { + w.violate("worker %d still counted as parked", wk.id) + } + } + if n := s.processorCount.Load(); n != 0 { + w.violate("processor count = %d after stop", n) + } + s.admitMu.Lock() + if s.admitting != 0 { + w.violate("admissions in flight = %d after stop", s.admitting) + } + s.admitMu.Unlock() + n := 0 + s.byPID.Range(func(_, _ any) bool { n++; return true }) + if n != 0 { + w.violate("%d processors still registered by pid", n) + } + n = 0 + s.byQueue.Range(func(_, _ any) bool { n++; return true }) + if n != 0 { + w.violate("%d processors still registered by queue", n) + } + if l := s.global.Len(); l > 0 { + w.violate("global queue holds %d entries after stop", l) + } + + if len(w.violations) > 0 { + t.Errorf("seed %d: %d invariant violations (replay: WIPPY_CHAOS_SEED=%d WIPPY_CHAOS_RUNS=1):\n %s", + seed, len(w.violations), seed, strings.Join(w.violations, "\n ")) + } +} + +// checkOutcome verifies the process ended the way its plan and the external +// events that hit it allow. +func (w *chaosWorld) checkOutcome(rec *chaosRec, res *apiruntime.Result) { + if res == nil { + w.violate("rec %d: no result", rec.id) + return + } + err := res.Error + killed := rec.killed.Load() || w.stopped.Load() + switch { + case err == nil: + var tok chaosToken + var ok bool + if res.Value != nil { + tok, ok = res.Value.Data().(chaosToken) + } + if !ok || tok.rec != rec.id || tok.inc != len(rec.plans)-1 { + w.violate("rec %d: result %v does not come from the final incarnation %d", rec.id, res.Value, len(rec.plans)-1) + } + last := rec.plans[len(rec.plans)-1] + if k := last[len(last)-1].kind; k == opFail || k == opNever { + w.violate("rec %d: finished although its plan ends with kind %d", rec.id, k) + } + case errors.Is(err, process.ErrStepLimitExceeded): + if !rec.expectsLimit() { + w.violate("rec %d: step limit error without max_steps", rec.id) + } + case errors.Is(err, errChaosFail): + last := rec.plans[len(rec.plans)-1] + if last[len(last)-1].kind != opFail { + w.violate("rec %d: planned failure but plan does not fail", rec.id) + } + case errors.Is(err, sysprocess.ErrTerminated), errors.Is(err, errChaosCancel), errors.Is(err, context.Canceled): + if !killed { + w.violate("rec %d: terminated without a terminate or stop (%v)", rec.id, err) + } + case errors.Is(err, errChaosStart): + if rec.failUpgrade == 0 && !rec.failUpInit { + w.violate("rec %d: upgrade failure without injected failure (%v)", rec.id, err) + } + default: + w.violate("rec %d: unexpected error %v", rec.id, err) + } + if err != nil && !killed && !rec.expectsLimit() && rec.failUpgrade == 0 && !rec.failUpInit { + last := rec.plans[len(rec.plans)-1] + if last[len(last)-1].kind != opFail { + w.violate("rec %d: unprovoked failure %v", rec.id, err) + } + } + if err == nil && rec.blocker && !killed { + w.violate("rec %d: blocker finished", rec.id) + } +} + +func outcomeName(res *apiruntime.Result) string { + switch err := res.Error; { + case err == nil: + return "done" + case errors.Is(err, process.ErrStepLimitExceeded): + return "limit" + case errors.Is(err, errChaosFail): + return "fail" + case errors.Is(err, errChaosStart): + return "upgrade-fail" + default: + return "killed" + } +} From 76395389c48daa60446e9008034a94c4c950ba8e Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sat, 3 Oct 2026 12:58:47 -0400 Subject: [PATCH 46/50] test: bound chaos message supply and satisfy lint --- api/runtime/lua/options_fuzz_test.go | 9 +- runtime/lua/engine/chaos_test.go | 87 ++++++++++--------- system/scheduler/actor/chaos_test.go | 123 +++++++++++++-------------- system/scheduler/actor/scheduler.go | 6 +- 4 files changed, 114 insertions(+), 111 deletions(-) diff --git a/api/runtime/lua/options_fuzz_test.go b/api/runtime/lua/options_fuzz_test.go index 49e2da918..d4f05f061 100644 --- a/api/runtime/lua/options_fuzz_test.go +++ b/api/runtime/lua/options_fuzz_test.go @@ -198,7 +198,7 @@ func addOptionSeeds(f *testing.F) { f.Add(kind, i, uint64(i), float64(i), "100", kind+1, i, uint64(i), float64(i), "7") } f.Add(kind, int64(5), uint64(math.MaxUint64), math.NaN(), "1.5", kind, int64(-5), uint64(1)<<63, math.Inf(1), "1e3") - f.Add(kind, int64(5), uint64(5), 0.5, "", kind, int64(5), uint64(5), -0.0, "99999999999999999999") + f.Add(kind, int64(5), uint64(5), 0.5, "", kind, int64(5), uint64(5), math.Copysign(0, -1), "99999999999999999999") } } @@ -290,9 +290,10 @@ func FuzzEntryExecutionBudgets(f *testing.F) { raw := scalars[int(outer)%len(scalars)] _, err = EntryExecutionBudgets(attrs.Bag{entryOptionsMetaKey: raw}) switch raw.(type) { - case map[string]any, attrs.Bag: - if err != nil { - t.Fatalf("map meta.options rejected: %v", err) + case map[string]any: + // The only map among the candidates holds the unknown key "v". + if err == nil { + t.Fatal("meta.options with an unknown field accepted") } case nil: if err != nil { diff --git a/runtime/lua/engine/chaos_test.go b/runtime/lua/engine/chaos_test.go index b4b71b305..c64033e24 100644 --- a/runtime/lua/engine/chaos_test.go +++ b/runtime/lua/engine/chaos_test.go @@ -37,7 +37,7 @@ import ( // // WIPPY_LUA_CHAOS_SEED first seed (default 1) // WIPPY_LUA_CHAOS_RUNS number of consecutive seeds (default 3, 100 with WIPPY_CHAOS_LONG) -// WIPPY_LUA_CHAOS_PROCS actors per run (default 40, 150 with WIPPY_CHAOS_LONG) +// WIPPY_LUA_CHAOS_PROCS actors per run (default 24, 150 with WIPPY_CHAOS_LONG) func luaChaosEnvInt(name string, short, long int) int { def := short if os.Getenv("WIPPY_CHAOS_LONG") != "" { @@ -66,26 +66,26 @@ type luaStage struct { type luaRec struct { rng *rand.Rand - pid pidapi.PID cancel context.CancelFunc spawn attrs.Bag - spawnBudget luaapi.ExecutionBudgets ready chan struct{} - stages []luaStage result *runtime.Result + pid pidapi.PID + stages []luaStage + spawnBudget luaapi.ExecutionBudgets id int - completes atomic.Int32 + resultMu sync.Mutex starts atomic.Int32 upgrades atomic.Int32 submitted atomic.Bool gaveUp atomic.Bool killed atomic.Bool running atomic.Int32 + completes atomic.Int32 blocker bool failing bool wantsMsgs bool wantsWake bool - resultMu sync.Mutex } // limited reports whether any incarnation of the actor can be stopped by a @@ -121,7 +121,7 @@ type luaChaos struct { stopped atomic.Bool } -func (w *luaChaos) violate(format string, args ...any) { +func (w *luaChaos) violatef(format string, args ...any) { w.mu.Lock() defer w.mu.Unlock() if len(w.violations) < 40 { @@ -149,21 +149,21 @@ func (p *chaosLua) Init(ctx context.Context, method string, input payload.Payloa func (p *chaosLua) Step(events []process.Event, out *process.StepOutput) error { if !p.running.CompareAndSwap(0, 1) { - p.w.violate("rec %d inc %d: concurrent steps of one incarnation", p.rec.id, p.inc) + p.w.violatef("rec %d inc %d: concurrent steps of one incarnation", p.rec.id, p.inc) } defer p.running.Store(0) if p.rec.running.Add(1) != 1 { - p.w.violate("rec %d: steps of two incarnations overlap", p.rec.id) + p.w.violatef("rec %d: steps of two incarnations overlap", p.rec.id) } defer p.rec.running.Add(-1) if p.rec.completes.Load() != 0 { - p.w.violate("rec %d inc %d: step after completion", p.rec.id, p.inc) + p.w.violatef("rec %d inc %d: step after completion", p.rec.id, p.inc) } if p.closed.Load() != 0 { - p.w.violate("rec %d inc %d: step after close", p.rec.id, p.inc) + p.w.violatef("rec %d inc %d: step after close", p.rec.id, p.inc) } if p.limited.Load() { - p.w.violate("rec %d inc %d: step after the step limit failed it", p.rec.id, p.inc) + p.w.violatef("rec %d inc %d: step after the step limit failed it", p.rec.id, p.inc) } err := p.Process.Step(events, out) @@ -171,21 +171,21 @@ func (p *chaosLua) Step(events []process.Event, out *process.StepOutput) error { case errors.Is(err, process.ErrStepLimitExceeded): p.limited.Store(true) if p.limit == 0 { - p.w.violate("rec %d inc %d: step limit error without max_steps", p.rec.id, p.inc) + p.w.violatef("rec %d inc %d: step limit error without max_steps", p.rec.id, p.inc) } - case p.limit > 0 && p.Process.StepsUsed() > p.limit: - p.w.violate("rec %d inc %d: %d steps used with max_steps %d (err %v)", p.rec.id, p.inc, p.Process.StepsUsed(), p.limit, err) + case p.limit > 0 && p.StepsUsed() > p.limit: + p.w.violatef("rec %d inc %d: %d steps used with max_steps %d (err %v)", p.rec.id, p.inc, p.StepsUsed(), p.limit, err) } return err } func (p *chaosLua) Close() { if p.closed.Add(1) != 1 { - p.w.violate("rec %d inc %d: closed more than once", p.rec.id, p.inc) + p.w.violatef("rec %d inc %d: closed more than once", p.rec.id, p.inc) return } if p.running.Load() != 0 { - p.w.violate("rec %d inc %d: closed during a step", p.rec.id, p.inc) + p.w.violatef("rec %d inc %d: closed during a step", p.rec.id, p.inc) } p.Process.Close() } @@ -200,7 +200,7 @@ func (f luaChaosFactory) Create(id registry.ID) (process.Process, *process.Meta, rec := f.w.recs[n] inc := int(rec.upgrades.Add(1)) if inc >= len(rec.stages) { - f.w.violate("rec %d: upgrade to incarnation %d but there are %d", rec.id, inc, len(rec.stages)) + f.w.violatef("rec %d: upgrade to incarnation %d but there are %d", rec.id, inc, len(rec.stages)) return nil, nil, errLuaChaosStart } p, err := f.w.newProc(rec, inc) @@ -270,7 +270,7 @@ func luaTriangle(n int) int { return n * (n + 1) / 2 } func luaStageBody(rng *rand.Rand, rec *luaRec) (body, expected string) { switch rng.Intn(6) { case 0: - n := 100 + rng.Intn(4000) + n := 100 + rng.Intn(2000) return fmt.Sprintf("local s = 0\nfor i = 1, %d do s = s + i end\nlocal result = s", n), strconv.Itoa(luaTriangle(n)) case 1: k, n, y := 2+rng.Intn(5), 50+rng.Intn(400), 1+rng.Intn(40) @@ -445,7 +445,7 @@ func (l luaChaosLifecycle) recFor(p pidapi.PID) *luaRec { func (l luaChaosLifecycle) OnStart(_ context.Context, p pidapi.PID, _ process.Process) error { if rec := l.recFor(p); rec != nil && rec.starts.Add(1) != 1 { - l.w.violate("rec %d: started more than once", rec.id) + l.w.violatef("rec %d: started more than once", rec.id) } return nil } @@ -456,10 +456,10 @@ func (l luaChaosLifecycle) OnComplete(_ context.Context, p pidapi.PID, res *runt return } if rec.running.Load() != 0 { - l.w.violate("rec %d: completed during a step", rec.id) + l.w.violatef("rec %d: completed during a step", rec.id) } if rec.completes.Add(1) != 1 { - l.w.violate("rec %d: completed more than once", rec.id) + l.w.violatef("rec %d: completed more than once", rec.id) return } rec.resultMu.Lock() @@ -470,18 +470,18 @@ func (l luaChaosLifecycle) OnComplete(_ context.Context, p pidapi.PID, res *runt func (w *luaChaos) submit(rec *luaRec) { proc, err := w.newProc(rec, 0) if err != nil { - w.violate("rec %d: cannot build the process: %v", rec.id, err) + w.violatef("rec %d: cannot build the process: %v", rec.id, err) rec.gaveUp.Store(true) close(rec.ready) return } frame, fc := ctxapi.OpenFrameContext(w.appCtx) if err := runtime.SetFrameID(frame, registry.NewID("chaos", strconv.Itoa(rec.id))); err != nil { - w.violate("rec %d: %v", rec.id, err) + w.violatef("rec %d: %v", rec.id, err) } if rec.spawn != nil { if err := fc.Set(runtime.FrameLifecycleOptionsKey, attrs.Attributes(rec.spawn)); err != nil { - w.violate("rec %d: %v", rec.id, err) + w.violatef("rec %d: %v", rec.id, err) } } ctx, cancel := context.WithCancel(frame) @@ -491,7 +491,7 @@ func (w *luaChaos) submit(rec *luaRec) { cancel() proc.Close() if !errors.Is(err, process.ErrSchedulerStopping) { - w.violate("rec %d: unexpected Submit error %v", rec.id, err) + w.violatef("rec %d: unexpected Submit error %v", rec.id, err) } rec.gaveUp.Store(true) close(rec.ready) @@ -511,7 +511,7 @@ func TestLuaChaos(t *testing.T) { } func runLuaChaos(t *testing.T, seed int64) { - procs := luaChaosEnvInt("WIPPY_LUA_CHAOS_PROCS", 40, 150) + procs := luaChaosEnvInt("WIPPY_LUA_CHAOS_PROCS", 24, 150) rng := rand.New(rand.NewSource(seed)) workers := 1 + rng.Intn(4) stopMode := rng.Intn(3) @@ -555,11 +555,14 @@ func runLuaChaos(t *testing.T, seed int64) { deadline := time.Now().Add(20 * time.Second) killAt := time.Now().Add(time.Duration(r.Intn(40)) * time.Millisecond) kill := rec.blocker || r.Intn(8) == 0 - for rec.completes.Load() == 0 && time.Now().Before(deadline) { + // Unconsumed messages stay queued in the actor, so the supply is bounded. + sent := 0 + for rec.completes.Load() == 0 && time.Now().Before(deadline) && sent < 1500 { + sent++ if rec.wantsMsgs { pkg := relay.NewPackage(pidapi.PID{}, rec.pid, "msg", payload.NewPayload(lua.LString("x"), payload.Lua)) if err := w.sched.Send(pkg); err != nil && !errors.Is(err, process.ErrProcessNotFound) && !errors.Is(err, process.ErrProcessClosed) { - w.violate("rec %d: unexpected Send error %v", rec.id, err) + w.violatef("rec %d: unexpected Send error %v", rec.id, err) } } if rec.wantsWake { @@ -571,11 +574,11 @@ func runLuaChaos(t *testing.T, seed int64) { if r.Intn(2) == 0 { rec.cancel() } else if err := w.sched.Terminate(rec.pid); err != nil && !errors.Is(err, process.ErrProcessNotFound) { - w.violate("rec %d: unexpected Terminate error %v", rec.id, err) + w.violatef("rec %d: unexpected Terminate error %v", rec.id, err) } return } - time.Sleep(time.Duration(100+r.Intn(900)) * time.Microsecond) + time.Sleep(time.Duration(300+r.Intn(700)) * time.Microsecond) } }() } @@ -598,7 +601,7 @@ func runLuaChaos(t *testing.T, seed int64) { wg.Wait() if stopMode == 0 { - w.awaitSettled(t, 15*time.Second) + w.awaitSettled(t, 60*time.Second) go startStop() } select { @@ -653,12 +656,12 @@ func (w *luaChaos) verify(t *testing.T, seed int64) { if rec.gaveUp.Load() { outcomes["rejected"]++ if rec.completes.Load() != 0 { - w.violate("rec %d: completed although admission failed", rec.id) + w.violatef("rec %d: completed although admission failed", rec.id) } continue } if n := rec.completes.Load(); n != 1 { - w.violate("rec %d: completed %d times", rec.id, n) + w.violatef("rec %d: completed %d times", rec.id, n) continue } rec.resultMu.Lock() @@ -673,16 +676,16 @@ func (w *luaChaos) verify(t *testing.T, seed int64) { w.mu.Unlock() for _, p := range incs { if p.inits.Load() > 0 && p.closed.Load() != 1 { - w.violate("rec %d inc %d: closed %d times, want once", p.rec.id, p.inc, p.closed.Load()) + w.violatef("rec %d inc %d: closed %d times, want once", p.rec.id, p.inc, p.closed.Load()) } } stats := w.sched.Stats() if stats["processes"] != 0 { - w.violate("%d processes still registered after stop", stats["processes"]) + w.violatef("%d processes still registered after stop", stats["processes"]) } if stats["global_queue"] != 0 { - w.violate("global queue holds %d entries after stop", stats["global_queue"]) + w.violatef("global queue holds %d entries after stop", stats["global_queue"]) } if len(w.violations) > 0 { t.Errorf("seed %d: %d invariant violations (replay: WIPPY_LUA_CHAOS_SEED=%d WIPPY_LUA_CHAOS_RUNS=1):\n %s", @@ -693,7 +696,7 @@ func (w *luaChaos) verify(t *testing.T, seed int64) { // checkOutcome verifies a finished actor and names its outcome class. func (w *luaChaos) checkOutcome(rec *luaRec, res *runtime.Result) string { if res == nil { - w.violate("rec %d: no result", rec.id) + w.violatef("rec %d: no result", rec.id) return "none" } killed := rec.killed.Load() || w.stopped.Load() @@ -707,16 +710,16 @@ func (w *luaChaos) checkOutcome(rec *luaRec, res *runtime.Result) string { } want := fmt.Sprintf("gen%d:%s", len(rec.stages)-1, last.expected) if rec.failing || rec.blocker { - w.violate("rec %d: finished with %q although its final stage cannot finish", rec.id, got) + w.violatef("rec %d: finished with %q although its final stage cannot finish", rec.id, got) } else if got != want { - w.violate("rec %d: result %q, want %q", rec.id, got, want) + w.violatef("rec %d: result %q, want %q", rec.id, got, want) } return "done" } switch { case errors.Is(res.Error, process.ErrStepLimitExceeded): if !rec.limited() { - w.violate("rec %d: step limit error without max_steps", rec.id) + w.violatef("rec %d: step limit error without max_steps", rec.id) } return "limit" case rec.failing && strings.Contains(res.Error.Error(), "chaos-boom"): @@ -724,6 +727,6 @@ func (w *luaChaos) checkOutcome(rec *luaRec, res *runtime.Result) string { case killed: return "killed" } - w.violate("rec %d: unexpected error %v", rec.id, res.Error) + w.violatef("rec %d: unexpected error %v", rec.id, res.Error) return "unexpected" } diff --git a/system/scheduler/actor/chaos_test.go b/system/scheduler/actor/chaos_test.go index f1f95daff..f610c70eb 100644 --- a/system/scheduler/actor/chaos_test.go +++ b/system/scheduler/actor/chaos_test.go @@ -118,29 +118,29 @@ type chaosOp struct { type chaosRec struct { result *apiruntime.Result rng *rand.Rand - plans [][]chaosOp - pid pidapi.PID cancelParent context.CancelFunc ready chan struct{} - maxSteps uint64 + pid pidapi.PID + plans [][]chaosOp + failUpgrade int need int64 id int stepsRun atomic.Uint64 msgs atomic.Int64 - completes atomic.Int32 - starts atomic.Int32 - running atomic.Int32 - upgrades atomic.Int32 + maxSteps uint64 + resultMu sync.Mutex killed atomic.Bool + upgrades atomic.Int32 + running atomic.Int32 submitted atomic.Bool gaveUp atomic.Bool - failUpgrade int + starts atomic.Int32 + completes atomic.Int32 failStart bool deaf bool blocker bool pooled bool failUpInit bool - resultMu sync.Mutex } func (r *chaosRec) expectsLimit() bool { return r.maxSteps > 0 } @@ -157,7 +157,7 @@ type chaosWorld struct { stopped atomic.Bool } -func (w *chaosWorld) violate(format string, args ...any) { +func (w *chaosWorld) violatef(format string, args ...any) { w.mu.Lock() defer w.mu.Unlock() if len(w.violations) < 40 { @@ -175,18 +175,18 @@ func (w *chaosWorld) track(p *chaosProc) { type chaosProc struct { w *chaosWorld rec *chaosRec - plan []chaosOp rng *rand.Rand outstanding map[uint64]struct{} - inits atomic.Int32 - closed atomic.Int32 - running atomic.Int32 + plan []chaosOp steps uint64 nextTag uint64 waitTag uint64 pc int ph int inc int + closed atomic.Int32 + running atomic.Int32 + inits atomic.Int32 preempt bool limited bool } @@ -203,7 +203,7 @@ func (p *chaosProc) ResumeStepCount(used uint64) { p.steps = used } func (p *chaosProc) Init(context.Context, string, payload.Payloads) error { if p.inits.Add(1) != 1 { - p.w.violate("rec %d inc %d: Init called more than once", p.rec.id, p.inc) + p.w.violatef("rec %d inc %d: Init called more than once", p.rec.id, p.inc) } if p.rec.failUpInit && p.inc > 0 { return errChaosStart @@ -213,10 +213,10 @@ func (p *chaosProc) Init(context.Context, string, payload.Payloads) error { func (p *chaosProc) Close() { if p.closed.Add(1) != 1 { - p.w.violate("rec %d inc %d: closed more than once", p.rec.id, p.inc) + p.w.violatef("rec %d inc %d: closed more than once", p.rec.id, p.inc) } if p.running.Load() != 0 { - p.w.violate("rec %d inc %d: closed during a step", p.rec.id, p.inc) + p.w.violatef("rec %d inc %d: closed during a step", p.rec.id, p.inc) } } @@ -232,22 +232,22 @@ func isCancelEvent(data any) bool { func (p *chaosProc) Step(events []process.Event, out *process.StepOutput) error { rec := p.rec if !p.running.CompareAndSwap(0, 1) { - p.w.violate("rec %d inc %d: concurrent steps of one incarnation", rec.id, p.inc) + p.w.violatef("rec %d inc %d: concurrent steps of one incarnation", rec.id, p.inc) } defer p.running.Store(0) if rec.running.Add(1) != 1 { - p.w.violate("rec %d: steps of two incarnations overlap", rec.id) + p.w.violatef("rec %d: steps of two incarnations overlap", rec.id) } defer rec.running.Add(-1) if rec.completes.Load() != 0 { - p.w.violate("rec %d inc %d: step after completion", rec.id, p.inc) + p.w.violatef("rec %d inc %d: step after completion", rec.id, p.inc) } if p.closed.Load() != 0 { - p.w.violate("rec %d inc %d: step after close", rec.id, p.inc) + p.w.violatef("rec %d inc %d: step after close", rec.id, p.inc) } if p.limited { - p.w.violate("rec %d inc %d: step after the step limit failed it", rec.id, p.inc) + p.w.violatef("rec %d inc %d: step after the step limit failed it", rec.id, p.inc) } p.steps++ @@ -257,7 +257,7 @@ func (p *chaosProc) Step(events []process.Event, out *process.StepOutput) error } total := rec.stepsRun.Add(1) if rec.maxSteps > 0 && total > rec.maxSteps { - p.w.violate("rec %d: %d steps ran with max_steps %d", rec.id, total, rec.maxSteps) + p.w.violatef("rec %d: %d steps ran with max_steps %d", rec.id, total, rec.maxSteps) } cancelled := false @@ -282,7 +282,7 @@ func (p *chaosProc) Step(events []process.Event, out *process.StepOutput) error func (p *chaosProc) absorb(e process.Event) { rec := p.rec if _, ok := p.outstanding[e.Tag]; !ok { - p.w.violate("rec %d inc %d: completion for tag %d that is not outstanding (data %#v)", rec.id, p.inc, e.Tag, e.Data) + p.w.violatef("rec %d inc %d: completion for tag %d that is not outstanding (data %#v)", rec.id, p.inc, e.Tag, e.Data) return } delete(p.outstanding, e.Tag) @@ -291,7 +291,7 @@ func (p *chaosProc) absorb(e process.Event) { } tok, ok := e.Data.(chaosToken) if !ok || tok.rec != rec.id || tok.inc != p.inc || tok.tag != e.Tag { - p.w.violate("rec %d inc %d: completion %#v delivered to the wrong yield (tag %d)", rec.id, p.inc, e.Data, e.Tag) + p.w.violatef("rec %d inc %d: completion %#v delivered to the wrong yield (tag %d)", rec.id, p.inc, e.Data, e.Tag) } } @@ -393,7 +393,7 @@ func (f chaosFactory) Create(id registry.ID) (process.Process, *process.Meta, er return nil, nil, errChaosStart } if inc >= len(rec.plans) { - f.w.violate("rec %d: upgrade to incarnation %d but the plan has %d", rec.id, inc, len(rec.plans)) + f.w.violatef("rec %d: upgrade to incarnation %d but the plan has %d", rec.id, inc, len(rec.plans)) return nil, nil, errChaosStart } return f.w.newProc(rec, inc), &process.Meta{Method: "main"}, nil @@ -445,7 +445,7 @@ func (l chaosLifecycle) OnStart(_ context.Context, p pidapi.PID, _ process.Proce return nil } if rec.starts.Add(1) != 1 { - l.w.violate("rec %d: started more than once", rec.id) + l.w.violatef("rec %d: started more than once", rec.id) } if rec.failStart { return errChaosStart @@ -459,10 +459,10 @@ func (l chaosLifecycle) OnComplete(_ context.Context, p pidapi.PID, res *apirunt return } if rec.running.Load() != 0 { - l.w.violate("rec %d: completed during a step", rec.id) + l.w.violatef("rec %d: completed during a step", rec.id) } if rec.completes.Add(1) != 1 { - l.w.violate("rec %d: completed more than once", rec.id) + l.w.violatef("rec %d: completed more than once", rec.id) return } rec.resultMu.Lock() @@ -522,13 +522,13 @@ func chaosPlanFor(rng *rand.Rand, rec *chaosRec) { plan = append(plan, chaosOp{kind: opUpgrade}) } } else { - switch { - case kind == 4: + switch kind { + case 4: plan = append(plan, chaosOp{kind: opFail}) - case kind == 5: + case 5: rec.blocker = true plan = append(plan, chaosOp{kind: opNever}) - case kind == 6: + case 6: rec.need = int64(50 + rng.Intn(400)) plan = append(plan, chaosOp{kind: opWaitMsg, n: int(rec.need)}, chaosOp{kind: opFinish}) default: @@ -572,10 +572,10 @@ func (w *chaosWorld) submit(rec *chaosRec) { switch { case errors.Is(err, process.ErrSchedulerStopping), errors.Is(err, process.ErrMaxProcessesExceeded), errors.Is(err, errChaosStart): default: - w.violate("rec %d: unexpected Submit error %v", rec.id, err) + w.violatef("rec %d: unexpected Submit error %v", rec.id, err) } if rec.completes.Load() != 0 { - w.violate("rec %d: completed although Submit failed", rec.id) + w.violatef("rec %d: completed although Submit failed", rec.id) } rec.gaveUp.Store(true) close(rec.ready) @@ -597,7 +597,7 @@ func (w *chaosWorld) submitPooled(rec *chaosRec) { if err != nil { proc.Close() if !errors.Is(err, process.ErrSchedulerStopping) && !errors.Is(err, process.ErrMaxProcessesExceeded) { - w.violate("rec %d: unexpected CreateProcessor error %v", rec.id, err) + w.violatef("rec %d: unexpected CreateProcessor error %v", rec.id, err) } rec.gaveUp.Store(true) close(rec.ready) @@ -616,7 +616,7 @@ func (w *chaosWorld) submitPooled(rec *chaosRec) { select { case <-results: case <-time.After(30 * time.Second): - w.violate("rec %d: pooled execution did not complete", rec.id) + w.violatef("rec %d: pooled execution did not complete", rec.id) } w.sched.ReleaseProcessor(pr) } @@ -628,7 +628,7 @@ func (w *chaosWorld) allowedSendError(err error) bool { func (w *chaosWorld) send(rec *chaosRec) { err := w.sched.Send(&relay.Package{Target: rec.pid, Messages: []*relay.Message{{Topic: "chaos"}}}) if !w.allowedSendError(err) { - w.violate("rec %d: unexpected Send error %v", rec.id, err) + w.violatef("rec %d: unexpected Send error %v", rec.id, err) } } @@ -697,7 +697,6 @@ func runSchedulerChaos(t *testing.T, seed int64) { var wg sync.WaitGroup for _, rec := range w.recs { - rec := rec wg.Add(1) go func() { defer wg.Done() @@ -729,7 +728,7 @@ func runSchedulerChaos(t *testing.T, seed int64) { if r.Intn(2) == 0 { rec.cancelParent() } else if err := w.sched.Terminate(rec.pid); err != nil && !errors.Is(err, process.ErrProcessNotFound) { - w.violate("rec %d: unexpected Terminate error %v", rec.id, err) + w.violatef("rec %d: unexpected Terminate error %v", rec.id, err) } } }() @@ -744,7 +743,7 @@ func runSchedulerChaos(t *testing.T, seed int64) { for i := 0; i < 20; i++ { time.Sleep(time.Duration(r.Intn(4000)) * time.Microsecond) if err := w.sched.ResizeWorkers(1 + r.Intn(6)); err != nil && !errors.Is(err, process.ErrSchedulerStopping) { - w.violate("unexpected ResizeWorkers error %v", err) + w.violatef("unexpected ResizeWorkers error %v", err) } } }() @@ -762,7 +761,7 @@ func runSchedulerChaos(t *testing.T, seed int64) { w.sched.Stop(ctx) cancel() if stopMode != 2 && time.Since(started) > stopTimeout-time.Second { - w.violate("Stop took %v, close to its %v deadline", time.Since(started), stopTimeout) + w.violatef("Stop took %v, close to its %v deadline", time.Since(started), stopTimeout) } close(stopDone) } @@ -831,12 +830,12 @@ func (w *chaosWorld) verify(t *testing.T, seed int64) { if rec.gaveUp.Load() { rejected++ if completes != 0 { - w.violate("rec %d: completed although admission failed", rec.id) + w.violatef("rec %d: completed although admission failed", rec.id) } continue } if completes != 1 { - w.violate("rec %d: completed %d times (starts %d)", rec.id, completes, starts) + w.violatef("rec %d: completed %d times (starts %d)", rec.id, completes, starts) continue } rec.resultMu.Lock() @@ -853,39 +852,39 @@ func (w *chaosWorld) verify(t *testing.T, seed int64) { w.mu.Unlock() for _, p := range incs { if p.inits.Load() > 0 && p.closed.Load() != 1 { - w.violate("rec %d inc %d: closed %d times, want once", p.rec.id, p.inc, p.closed.Load()) + w.violatef("rec %d inc %d: closed %d times, want once", p.rec.id, p.inc, p.closed.Load()) } } s := w.sched if n := s.parked.Load(); n != 0 { - w.violate("parked counter = %d after stop", n) + w.violatef("parked counter = %d after stop", n) } for _, wk := range s.workerSnapshot() { if wk.parked.Load() { - w.violate("worker %d still counted as parked", wk.id) + w.violatef("worker %d still counted as parked", wk.id) } } if n := s.processorCount.Load(); n != 0 { - w.violate("processor count = %d after stop", n) + w.violatef("processor count = %d after stop", n) } s.admitMu.Lock() if s.admitting != 0 { - w.violate("admissions in flight = %d after stop", s.admitting) + w.violatef("admissions in flight = %d after stop", s.admitting) } s.admitMu.Unlock() n := 0 s.byPID.Range(func(_, _ any) bool { n++; return true }) if n != 0 { - w.violate("%d processors still registered by pid", n) + w.violatef("%d processors still registered by pid", n) } n = 0 s.byQueue.Range(func(_, _ any) bool { n++; return true }) if n != 0 { - w.violate("%d processors still registered by queue", n) + w.violatef("%d processors still registered by queue", n) } if l := s.global.Len(); l > 0 { - w.violate("global queue holds %d entries after stop", l) + w.violatef("global queue holds %d entries after stop", l) } if len(w.violations) > 0 { @@ -898,7 +897,7 @@ func (w *chaosWorld) verify(t *testing.T, seed int64) { // events that hit it allow. func (w *chaosWorld) checkOutcome(rec *chaosRec, res *apiruntime.Result) { if res == nil { - w.violate("rec %d: no result", rec.id) + w.violatef("rec %d: no result", rec.id) return } err := res.Error @@ -911,40 +910,40 @@ func (w *chaosWorld) checkOutcome(rec *chaosRec, res *apiruntime.Result) { tok, ok = res.Value.Data().(chaosToken) } if !ok || tok.rec != rec.id || tok.inc != len(rec.plans)-1 { - w.violate("rec %d: result %v does not come from the final incarnation %d", rec.id, res.Value, len(rec.plans)-1) + w.violatef("rec %d: result %v does not come from the final incarnation %d", rec.id, res.Value, len(rec.plans)-1) } last := rec.plans[len(rec.plans)-1] if k := last[len(last)-1].kind; k == opFail || k == opNever { - w.violate("rec %d: finished although its plan ends with kind %d", rec.id, k) + w.violatef("rec %d: finished although its plan ends with kind %d", rec.id, k) } case errors.Is(err, process.ErrStepLimitExceeded): if !rec.expectsLimit() { - w.violate("rec %d: step limit error without max_steps", rec.id) + w.violatef("rec %d: step limit error without max_steps", rec.id) } case errors.Is(err, errChaosFail): last := rec.plans[len(rec.plans)-1] if last[len(last)-1].kind != opFail { - w.violate("rec %d: planned failure but plan does not fail", rec.id) + w.violatef("rec %d: planned failure but plan does not fail", rec.id) } case errors.Is(err, sysprocess.ErrTerminated), errors.Is(err, errChaosCancel), errors.Is(err, context.Canceled): if !killed { - w.violate("rec %d: terminated without a terminate or stop (%v)", rec.id, err) + w.violatef("rec %d: terminated without a terminate or stop (%v)", rec.id, err) } case errors.Is(err, errChaosStart): if rec.failUpgrade == 0 && !rec.failUpInit { - w.violate("rec %d: upgrade failure without injected failure (%v)", rec.id, err) + w.violatef("rec %d: upgrade failure without injected failure (%v)", rec.id, err) } default: - w.violate("rec %d: unexpected error %v", rec.id, err) + w.violatef("rec %d: unexpected error %v", rec.id, err) } if err != nil && !killed && !rec.expectsLimit() && rec.failUpgrade == 0 && !rec.failUpInit { last := rec.plans[len(rec.plans)-1] if last[len(last)-1].kind != opFail { - w.violate("rec %d: unprovoked failure %v", rec.id, err) + w.violatef("rec %d: unprovoked failure %v", rec.id, err) } } if err == nil && rec.blocker && !killed { - w.violate("rec %d: blocker finished", rec.id) + w.violatef("rec %d: blocker finished", rec.id) } } diff --git a/system/scheduler/actor/scheduler.go b/system/scheduler/actor/scheduler.go index 8de05f925..c84312f38 100644 --- a/system/scheduler/actor/scheduler.go +++ b/system/scheduler/actor/scheduler.go @@ -399,7 +399,7 @@ func (s *Scheduler) Submit(ctx context.Context, pid pid.PID, p process.Process, } } - s.wakeOnCancel(proc, procCtx, cancel) + s.wakeOnCancel(procCtx, proc, cancel) // A pooled processor can still be referenced by a stale queue entry from // its previous incarnation; Ready is published only once it is initialized. @@ -413,7 +413,7 @@ func (s *Scheduler) Submit(ctx context.Context, pid pid.PID, p process.Process, // wakeOnCancel makes cancellation of the incarnation's context wake an actor // parked without incoming messages. It captures only queue identity and // generation, because Processor objects are pooled. -func (s *Scheduler) wakeOnCancel(proc *Processor, ctx context.Context, cancel context.CancelFunc) { +func (s *Scheduler) wakeOnCancel(ctx context.Context, proc *Processor, cancel context.CancelFunc) { q, gen := proc.queue, proc.gen.Load() stopWake := context.AfterFunc(ctx, func() { if q.Push(process.Event{Type: process.EventMessage}, gen) { @@ -534,7 +534,7 @@ func (s *Scheduler) CreateProcessor(ctx context.Context, pid pid.PID, p process. s.processorCount.Add(1) s.byPID.Store(pid.String(), proc) s.byQueue.Store(proc.queue, proc) - s.wakeOnCancel(proc, procCtx, cancel) + s.wakeOnCancel(procCtx, proc, cancel) // Ready is published only once the processor is initialized; see Submit. proc.state.Store(int32(StateReady)) From 77184054fc4a9009f860718b1847c6f4a21bdddd Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sat, 3 Oct 2026 13:07:32 -0400 Subject: [PATCH 47/50] test: back off chaos message rate instead of capping supply --- runtime/lua/engine/chaos_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/runtime/lua/engine/chaos_test.go b/runtime/lua/engine/chaos_test.go index c64033e24..4cf0d8a31 100644 --- a/runtime/lua/engine/chaos_test.go +++ b/runtime/lua/engine/chaos_test.go @@ -555,9 +555,9 @@ func runLuaChaos(t *testing.T, seed int64) { deadline := time.Now().Add(20 * time.Second) killAt := time.Now().Add(time.Duration(r.Intn(40)) * time.Millisecond) kill := rec.blocker || r.Intn(8) == 0 - // Unconsumed messages stay queued in the actor, so the supply is bounded. + // Unconsumed messages stay queued in the actor, so the send rate backs off. sent := 0 - for rec.completes.Load() == 0 && time.Now().Before(deadline) && sent < 1500 { + for rec.completes.Load() == 0 && time.Now().Before(deadline) { sent++ if rec.wantsMsgs { pkg := relay.NewPackage(pidapi.PID{}, rec.pid, "msg", payload.NewPayload(lua.LString("x"), payload.Lua)) @@ -578,7 +578,7 @@ func runLuaChaos(t *testing.T, seed int64) { } return } - time.Sleep(time.Duration(300+r.Intn(700)) * time.Microsecond) + time.Sleep(time.Duration(300+r.Intn(700)) * time.Microsecond * time.Duration(1+min(sent/50, 30))) } }() } From 78ea9cfda5e60f1c7224f50e94f82513744345a6 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sat, 3 Oct 2026 13:42:08 -0400 Subject: [PATCH 48/50] chore(deps): go-lua with matcher limits and differential-test fixes --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 9ba8e8802..d6b2478b6 100644 --- a/go.mod +++ b/go.mod @@ -69,7 +69,7 @@ require ( github.com/tree-sitter/tree-sitter-php v0.25.0 github.com/tree-sitter/tree-sitter-python v0.25.0 github.com/tree-sitter/tree-sitter-typescript v0.23.2 - github.com/wippyai/go-lua v1.6.4-0.20261003145456-1a12b5df98d5 + github.com/wippyai/go-lua v1.6.4-0.20261003172345-af635159323f github.com/wippyai/module-registry-proto-go v0.0.2-0.20260908140534-f6e2910c835f github.com/wippyai/tree-sitter-markdown v0.0.3 github.com/wippyai/tree-sitter-sql v0.0.4 diff --git a/go.sum b/go.sum index e198b37a0..5d7293f2a 100644 --- a/go.sum +++ b/go.sum @@ -556,6 +556,8 @@ github.com/wippyai/go-lua v1.6.4-0.20261003135019-e6c9f2e920e8 h1:cPF3sBY4gbJi5w github.com/wippyai/go-lua v1.6.4-0.20261003135019-e6c9f2e920e8/go.mod h1:cwD+390gJgx9cs+Zamby/lch5csqPV9RN1aDEfx/27M= github.com/wippyai/go-lua v1.6.4-0.20261003145456-1a12b5df98d5 h1:sJpAq4jGwV5q/co5qPnfi981X9dPkDKh8bdPGf068Aw= github.com/wippyai/go-lua v1.6.4-0.20261003145456-1a12b5df98d5/go.mod h1:cwD+390gJgx9cs+Zamby/lch5csqPV9RN1aDEfx/27M= +github.com/wippyai/go-lua v1.6.4-0.20261003172345-af635159323f h1:HBoZvymdzm4yklVwPAJlJIREypgRRcUeMNvdoKtIVdM= +github.com/wippyai/go-lua v1.6.4-0.20261003172345-af635159323f/go.mod h1:cwD+390gJgx9cs+Zamby/lch5csqPV9RN1aDEfx/27M= github.com/wippyai/module-registry-proto-go v0.0.2-0.20260908140534-f6e2910c835f h1:AoKS058roqnsHXaJ8Gww7ODz14qJqOK5QBeoALvcq6I= github.com/wippyai/module-registry-proto-go v0.0.2-0.20260908140534-f6e2910c835f/go.mod h1:p4ihYQKRQuqRVLxaH1YL0IEnkz4zKdyfVADOwvA5Tno= github.com/wippyai/tree-sitter-markdown v0.0.3 h1:u6e53+hzPSS848Xhm+I48SwtvWQHrC21wDrogdwCwuc= From 307dd77c5cab09ffd44b373dd44c46265aea213e Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sat, 3 Oct 2026 16:52:10 -0400 Subject: [PATCH 49/50] chore(deps): go-lua with overflow-safe integer numeric for --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index d6b2478b6..9b4b80d81 100644 --- a/go.mod +++ b/go.mod @@ -69,7 +69,7 @@ require ( github.com/tree-sitter/tree-sitter-php v0.25.0 github.com/tree-sitter/tree-sitter-python v0.25.0 github.com/tree-sitter/tree-sitter-typescript v0.23.2 - github.com/wippyai/go-lua v1.6.4-0.20261003172345-af635159323f + github.com/wippyai/go-lua v1.6.4-0.20261003205127-c564eb385ccd github.com/wippyai/module-registry-proto-go v0.0.2-0.20260908140534-f6e2910c835f github.com/wippyai/tree-sitter-markdown v0.0.3 github.com/wippyai/tree-sitter-sql v0.0.4 diff --git a/go.sum b/go.sum index 5d7293f2a..a85d58eb3 100644 --- a/go.sum +++ b/go.sum @@ -558,6 +558,8 @@ github.com/wippyai/go-lua v1.6.4-0.20261003145456-1a12b5df98d5 h1:sJpAq4jGwV5q/c github.com/wippyai/go-lua v1.6.4-0.20261003145456-1a12b5df98d5/go.mod h1:cwD+390gJgx9cs+Zamby/lch5csqPV9RN1aDEfx/27M= github.com/wippyai/go-lua v1.6.4-0.20261003172345-af635159323f h1:HBoZvymdzm4yklVwPAJlJIREypgRRcUeMNvdoKtIVdM= github.com/wippyai/go-lua v1.6.4-0.20261003172345-af635159323f/go.mod h1:cwD+390gJgx9cs+Zamby/lch5csqPV9RN1aDEfx/27M= +github.com/wippyai/go-lua v1.6.4-0.20261003205127-c564eb385ccd h1:Y6q7oiqgg7pKiV3A026Foiaqi/tvuvxdYn3sGZYGrgE= +github.com/wippyai/go-lua v1.6.4-0.20261003205127-c564eb385ccd/go.mod h1:cwD+390gJgx9cs+Zamby/lch5csqPV9RN1aDEfx/27M= github.com/wippyai/module-registry-proto-go v0.0.2-0.20260908140534-f6e2910c835f h1:AoKS058roqnsHXaJ8Gww7ODz14qJqOK5QBeoALvcq6I= github.com/wippyai/module-registry-proto-go v0.0.2-0.20260908140534-f6e2910c835f/go.mod h1:p4ihYQKRQuqRVLxaH1YL0IEnkz4zKdyfVADOwvA5Tno= github.com/wippyai/tree-sitter-markdown v0.0.3 h1:u6e53+hzPSS848Xhm+I48SwtvWQHrC21wDrogdwCwuc= From ebc4848d4e1bcaa143a08c19959cff7a59947de0 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Mon, 5 Oct 2026 09:55:08 -0400 Subject: [PATCH 50/50] test(actor): assert rejected admission cleanup using canonical PID key --- system/scheduler/actor/stop_admission_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/system/scheduler/actor/stop_admission_test.go b/system/scheduler/actor/stop_admission_test.go index cc3d0a95b..da03fa8fb 100644 --- a/system/scheduler/actor/stop_admission_test.go +++ b/system/scheduler/actor/stop_admission_test.go @@ -220,9 +220,10 @@ func TestLateRejectedAdmissionKeepsCallerOwnership(t *testing.T) { sched.lifecycle = lc sched.Start() p := &closeCountingProcess{} + self := pidapi.PID{UniqID: "late-rejection"} submitted := make(chan error, 1) go func() { - _, err := sched.Submit(context.Background(), pidapi.PID{UniqID: "late-rejection"}, p, "", nil) + _, err := sched.Submit(context.Background(), self, p, "", nil) submitted <- err }() <-lc.entered @@ -249,7 +250,7 @@ func TestLateRejectedAdmissionKeepsCallerOwnership(t *testing.T) { if sched.processorCount.Load() != 0 { t.Fatal("rejected admission retained process count") } - if _, ok := sched.byPID.Load("late-rejection"); ok { + if _, ok := sched.byPID.Load(self.String()); ok { t.Fatal("rejected admission retained routing") } sched.admitMu.Lock()