diff --git a/api/process/errors.go b/api/process/errors.go index 2658d08be..9de9b6999 100644 --- a/api/process/errors.go +++ b/api/process/errors.go @@ -20,6 +20,9 @@ const ( var ( ErrMaxProcessesExceeded = apierror.New(LimitExceeded, "max processes limit exceeded").WithRetryable(apierror.False) + // ErrStepLimitExceeded fails a process that exceeds its max_steps option. + ErrStepLimitExceeded = apierror.New(apierror.RateLimited, "process step limit exceeded").WithRetryable(apierror.False) + ErrProcessClosed = apierror.New(InvalidState, "process closed").WithRetryable(apierror.False) ErrProcessNotFound = apierror.New(NotFound, "process not found").WithRetryable(apierror.False) diff --git a/api/process/preempt.go b/api/process/preempt.go new file mode 100644 index 000000000..1c785058e --- /dev/null +++ b/api/process/preempt.go @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: MPL-2.0 + +package process + +// Preemptible is an optional interface for processes that can suspend a +// running step at a safepoint once they have used up their execution slice. +// A scheduler calls EnablePreemption once, when it admits the process, and +// the opt-in persists for the process's lifetime. It then runs the process +// again after Step reports StepPreempted; a process that was never enabled +// runs each step to a blocking point or completion. +type Preemptible interface { + EnablePreemption() +} + +// StepAccounted is an optional interface for processes that count their +// scheduler steps against a limit. A scheduler carries the count from a +// process to its upgrade replacement so the limit covers the whole lifetime +// of the actor rather than each incarnation. +type StepAccounted interface { + // StepsUsed returns the number of steps taken so far. + StepsUsed() uint64 + // ResumeStepCount sets the number of steps already taken. It is called + // after Init, before the first Step. + ResumeStepCount(used uint64) +} diff --git a/api/process/process_test.go b/api/process/process_test.go index 3e2eacc1f..16afed8a8 100644 --- a/api/process/process_test.go +++ b/api/process/process_test.go @@ -511,3 +511,21 @@ func (g *stubPIDGenerator) Generate(host pid.HostID) pid.PID { result.Host = host return result } + +type overflowCmd struct{ id dispatcher.CommandID } + +func (c overflowCmd) CmdID() dispatcher.CommandID { return c.id } + +// A reset output keeps no command references in its overflow buffer. +func TestStepOutputResetClearsOverflowYields(t *testing.T) { + var out StepOutput + for i := 0; i < MaxYields+3; i++ { + out.Yield(overflowCmd{id: dispatcher.CommandID(i)}, uint64(i)) + } + out.Reset() + for i, y := range out.ext[:cap(out.ext)] { + if y.Cmd != nil { + t.Fatalf("overflow yield %d still references its command", i) + } + } +} diff --git a/api/process/queue.go b/api/process/queue.go index b40dfa002..cb7945452 100644 --- a/api/process/queue.go +++ b/api/process/queue.go @@ -40,8 +40,12 @@ type EventQueue struct { events []Event drainBuf []Event generation atomic.Uint64 - mu sync.Mutex - closed atomic.Bool + // yieldEpoch identifies the code incarnation whose yield tags are valid. + // It changes when a process is replaced in place, so completions of the + // replaced code cannot resume the replacement. + yieldEpoch atomic.Uint64 + mu sync.Mutex + closed atomic.Bool } // messageTopicState is the accounting identity for one bounded topic @@ -120,6 +124,12 @@ func (q *EventQueue) Push(e Event, gen uint64) bool { // PushWithError preserves admission errors such as a full bounded mailbox. func (q *EventQueue) PushWithError(e Event, gen uint64) error { + return q.push(e, gen, nil) +} + +// push admits e when the generation, and the yield epoch if one is given, +// still match. +func (q *EventQueue) push(e Event, gen uint64, epoch *uint64) error { // Fast path: check generation and closed without lock if q.generation.Load() != gen { return ErrProcessClosed @@ -130,7 +140,7 @@ func (q *EventQueue) PushWithError(e Event, gen uint64) error { q.mu.Lock() // Recheck under lock - if q.generation.Load() != gen || q.closed.Load() { + if q.generation.Load() != gen || q.closed.Load() || (epoch != nil && q.yieldEpoch.Load() != *epoch) { q.mu.Unlock() return ErrProcessClosed } @@ -147,6 +157,26 @@ func (q *EventQueue) PushWithError(e Event, gen uint64) error { return nil } +// RetireYieldCompletions starts a new yield epoch for a process replaced in +// place: completers created before the call become inert and yield +// completions already queued are discarded. Messages and senders bound to the +// queue generation are unaffected. +func (q *EventQueue) RetireYieldCompletions() { + q.mu.Lock() + q.yieldEpoch.Add(1) + kept := q.events[:0] + for _, e := range q.events { + if e.Type == EventYieldComplete { + discardEvent(e) + } else { + kept = append(kept, e) + } + } + clear(q.events[len(kept):]) + q.events = kept + q.mu.Unlock() +} + // SetAdmission installs an execution's ingress policy before publishing its // queue to senders. Reset removes the old policy before a queue is reused. func (q *EventQueue) SetAdmission(admission EventAdmission) { @@ -523,16 +553,20 @@ func (q *EventQueue) discardPending() { for i := range q.events { e := &q.events[i] q.retireEventTopicsLocked(*e) - if d, ok := e.Data.(EventDiscarder); ok { - d.DiscardEvent() - } else if e.Type == EventMessage { - if pkg, ok := e.Data.(*relay.Package); ok { - relay.ReleasePackage(pkg) - } - } + discardEvent(*e) *e = Event{} } } + +func discardEvent(e Event) { + if d, ok := e.Data.(EventDiscarder); ok { + d.DiscardEvent() + } else if e.Type == EventMessage { + if pkg, ok := e.Data.(*relay.Package); ok { + relay.ReleasePackage(pkg) + } + } +} func (q *EventQueue) retireEventTopicsLocked(event Event) { if event.Type != EventMessage { return @@ -560,20 +594,23 @@ type YieldScheduler interface { } // YieldCompleter delivers yield completion results to a queue. -// Bound to a specific generation - stale completers silently no-op. +// Bound to a specific generation and yield epoch - stale completers silently no-op. // This breaks the direct reference to Processor, preventing races // when processor is released to pool while handler goroutines are still running. type YieldCompleter struct { scheduler YieldScheduler queue *EventQueue gen uint64 + epoch uint64 } -// NewYieldCompleter creates a completer bound to current queue generation. +// NewYieldCompleter creates a completer bound to the current queue generation +// and yield epoch. func (q *EventQueue) NewYieldCompleter(sched YieldScheduler) *YieldCompleter { return &YieldCompleter{ queue: q, gen: q.generation.Load(), + epoch: q.yieldEpoch.Load(), scheduler: sched, } } @@ -581,12 +618,12 @@ func (q *EventQueue) NewYieldCompleter(sched YieldScheduler) *YieldCompleter { // CompleteYield implements dispatcher.ResultReceiver. // Safe to call from any goroutine - uses generation to detect staleness. func (c *YieldCompleter) CompleteYield(tag uint64, data any, err error) { - if !c.queue.Push(Event{ + if c.queue.push(Event{ Type: EventYieldComplete, Tag: tag, Data: data, Error: err, - }, c.gen) { + }, c.gen, &c.epoch) != nil { return } diff --git a/api/process/queue_generation_test.go b/api/process/queue_generation_test.go index 2f4bec724..775d50a32 100644 --- a/api/process/queue_generation_test.go +++ b/api/process/queue_generation_test.go @@ -100,3 +100,22 @@ func TestA05ResetDrainsStaleSignal(t *testing.T) { default: } } + +func TestRetireYieldCompletionsInvalidatesEarlierCompleters(t *testing.T) { + queue := NewEventQueue() + wake := &boundaryWakeRecorder{} + old := queue.NewYieldCompleter(wake) + old.CompleteYield(1, "queued-before", nil) + require.True(t, queue.PushDirect(Event{Type: EventMessage, Data: "message"})) + + queue.RetireYieldCompletions() + fresh := queue.NewYieldCompleter(wake) + old.CompleteYield(1, "stale", nil) + fresh.CompleteYield(1, "fresh", nil) + + events := queue.Drain() + require.Len(t, events, 2) + assert.Equal(t, EventMessage, events[0].Type, "messages survive") + assert.Equal(t, "fresh", events[1].Data) + assert.Equal(t, 2, wake.calls, "the retired completer does not wake") +} diff --git a/api/process/queue_retirement_test.go b/api/process/queue_retirement_test.go new file mode 100644 index 000000000..e84d70db9 --- /dev/null +++ b/api/process/queue_retirement_test.go @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: MPL-2.0 + +package process + +import "testing" + +type ownedCompletion struct{ discarded int } + +func (c *ownedCompletion) DiscardEvent() { c.discarded++ } + +func TestRetiredCompletionsReleaseOwnership(t *testing.T) { + q := NewEventQueue() + first, second, message := &ownedCompletion{}, &ownedCompletion{}, &ownedCompletion{} + gen := q.Generation() + for _, e := range []Event{ + {Type: EventYieldComplete, Tag: 1, Data: first}, + {Type: EventMessage, Data: message}, + {Type: EventYieldComplete, Tag: 2, Data: second}, + } { + if !q.Push(e, gen) { + t.Fatal("queue rejected event") + } + } + q.RetireYieldCompletions() + q.RetireYieldCompletions() + if first.discarded != 1 || second.discarded != 1 { + t.Fatalf("retired ownership: %d, %d; want once each", first.discarded, second.discarded) + } + if message.discarded != 0 || q.Generation() != gen { + t.Fatal("retirement changed message ownership or queue generation") + } + q.Close() + if first.discarded != 1 || second.discarded != 1 || message.discarded != 1 { + t.Fatal("close must discard only the retained message") + } +} diff --git a/api/process/step.go b/api/process/step.go index 44e9d3c0c..1e85333d7 100644 --- a/api/process/step.go +++ b/api/process/step.go @@ -15,6 +15,10 @@ const ( StepDone StepYield // has yields to dispatch, wait for completions StepUpgrade // process requested upgrade, worker handles swap + // StepPreempted reports that the process used up its preemption budget + // and is ready to continue. The scheduler dispatches the step's yields + // and runs the process again after other ready work. + StepPreempted ) // MaxYields is the maximum yields per step that fit in the fixed buffer. @@ -97,10 +101,17 @@ func (o *StepOutput) Continue() { o.status = StepContinue } +// Preempt marks process as preempted: ready to continue once other ready +// work has had a turn. +func (o *StepOutput) Preempt() { + o.status = StepPreempted +} + // Reset clears output for reuse. func (o *StepOutput) Reset() { o.buf[0] = Yield{} o.buf[1] = Yield{} + clear(o.ext) o.ext = o.ext[:0] o.count = 0 o.status = StepContinue diff --git a/api/runtime/context.go b/api/runtime/context.go index bf4786cbf..57854ea41 100644 --- a/api/runtime/context.go +++ b/api/runtime/context.go @@ -73,6 +73,16 @@ func GetFramePID(ctx context.Context) (pid.PID, bool) { return pid.PID{}, false } +// SetFrameLifecycleOptions sets the lifecycle options (attrs.Attributes) in +// the FrameContext. Returns error if no frame context or frame is sealed. +func SetFrameLifecycleOptions(ctx context.Context, opts any) error { + fc := ctxapi.FrameFromContext(ctx) + if fc == nil { + return ctxapi.ErrNoFrameContext + } + return fc.Set(FrameLifecycleOptionsKey, opts) +} + // GetFrameLifecycleOptions retrieves lifecycle options from the FrameContext. // Returns nil if not found. func GetFrameLifecycleOptions(ctx context.Context) any { diff --git a/api/runtime/lua/options.go b/api/runtime/lua/options.go new file mode 100644 index 000000000..f98ac8190 --- /dev/null +++ b/api/runtime/lua/options.go @@ -0,0 +1,232 @@ +// SPDX-License-Identifier: MPL-2.0 + +package lua + +import ( + "encoding/json" + "fmt" + "math" + "sort" + "strconv" + + "github.com/wippyai/runtime/api/attrs" +) + +// Actor execution option keys, shared by process entry meta.options and +// process spawn options. +const ( + ProcessOptionTickBudget = "tick_budget" + ProcessOptionMaxSteps = "max_steps" +) + +// entryOptionsMetaKey is the entry meta bag holding declaration-time options. +const entryOptionsMetaKey = "options" + +// Process entry option keys other than the execution limits. They are +// accepted in meta.options so declarations written for the v2 runtime load +// unchanged; only the execution limits are read here. +const ( + ProcessOptionMailboxCapacity = "mailbox_capacity" + ProcessOptionMemoryLimitBytes = "memory_limit_bytes" + ProcessOptionHeapReserveBytes = "heap_reserve_bytes" + ProcessOptionHotPolicyID = "hot_policy_id" + ProcessOptionNetwork = "network" + // ProcessOptionDefaultHost names the host that serves the entry as a + // function; the process function listener reads it from meta.options. + ProcessOptionDefaultHost = "default_host" +) + +// IsProcessEntryOptionKey reports whether key is accepted in the meta.options +// of a process entry. +func IsProcessEntryOptionKey(key string) bool { + switch key { + case ProcessOptionTickBudget, ProcessOptionMaxSteps, ProcessOptionMailboxCapacity, + ProcessOptionMemoryLimitBytes, ProcessOptionHeapReserveBytes, ProcessOptionHotPolicyID, + ProcessOptionNetwork, ProcessOptionDefaultHost: + return true + } + return false +} + +// ExecutionBudgets holds the execution limits of a Lua actor. +// +// TickBudget is the number of VM ticks (backward jumps, loop iterations and +// calls) an actor may run before it is preempted so other actors can run: +// zero selects the runtime default and a negative value disables preemption. +// MaxSteps bounds the number of scheduler steps the actor may take; zero is +// unlimited and exceeding it fails the actor with process.ErrStepLimitExceeded. +type ExecutionBudgets struct { + TickBudget int64 + MaxSteps uint64 + // TickBudgetSet and MaxStepsSet report that the option was given, so an + // explicit zero overrides a lower-precedence value. + TickBudgetSet bool + MaxStepsSet bool +} + +// Override returns b with the options set in o taking precedence. +func (b ExecutionBudgets) Override(o ExecutionBudgets) ExecutionBudgets { + if o.TickBudgetSet { + b.TickBudget, b.TickBudgetSet = o.TickBudget, true + } + if o.MaxStepsSet { + b.MaxSteps, b.MaxStepsSet = o.MaxSteps, true + } + return b +} + +// ExecutionBudgetsFromOptions reads the execution options from an options +// bag; nil options hold no execution options. subject names the source in +// error messages. +func ExecutionBudgetsFromOptions(options attrs.Attributes, subject string) (ExecutionBudgets, error) { + var b ExecutionBudgets + if options == nil { + return b, nil + } + if raw, ok := options.Get(ProcessOptionTickBudget); ok && raw != nil { + v, err := optionInt64(raw) + if err != nil { + return b, fmt.Errorf("%s %q must be an integer", subject, ProcessOptionTickBudget) + } + b.TickBudget, b.TickBudgetSet = v, true + } + if raw, ok := options.Get(ProcessOptionMaxSteps); ok && raw != nil { + v, err := optionUint64(raw) + if err != nil { + return b, fmt.Errorf("%s %q must be a non-negative integer", subject, ProcessOptionMaxSteps) + } + b.MaxSteps, b.MaxStepsSet = v, true + } + return b, nil +} + +// EntryExecutionBudgets reads the execution options of a process entry from +// meta.options, which must be a map when present. The options are only accepted nested under meta.options. +func EntryExecutionBudgets(meta attrs.Bag) (ExecutionBudgets, error) { + for _, key := range [...]string{ProcessOptionTickBudget, ProcessOptionMaxSteps} { + if _, ok := meta.Get(key); ok { + return ExecutionBudgets{}, fmt.Errorf("process entry meta %q must be nested under meta.%s", key, entryOptionsMetaKey) + } + } + options, ok := meta.GetBag(entryOptionsMetaKey) + if !ok { + if raw, present := meta.Get(entryOptionsMetaKey); present && raw != nil { + return ExecutionBudgets{}, fmt.Errorf("process entry meta.%s must be a map, got %T", entryOptionsMetaKey, raw) + } + return ExecutionBudgets{}, nil + } + for _, key := range sortedKeys(options) { + if !IsProcessEntryOptionKey(key) { + return ExecutionBudgets{}, fmt.Errorf("process entry meta.%s has unknown field %q", entryOptionsMetaKey, key) + } + } + return ExecutionBudgetsFromOptions(options, "process entry meta.options") +} + +func sortedKeys(options attrs.Bag) []string { + keys := make([]string, 0, len(options)) + for key := range options { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} + +// maxExactFloatInteger is the largest integer a float64 holds exactly. +const maxExactFloatInteger = 1 << 53 + +func optionInt64(raw any) (int64, error) { + switch v := raw.(type) { + case int: + return int64(v), nil + case int8: + return int64(v), nil + case int16: + return int64(v), nil + case int32: + return int64(v), nil + case int64: + return v, nil + case uint: + return int64FromUint64(uint64(v)) + case uint8: + return int64(v), nil + case uint16: + return int64(v), nil + case uint32: + return int64(v), nil + case uint64: + return int64FromUint64(v) + case uintptr: + return int64FromUint64(uint64(v)) + case float32: + return int64FromFloat(float64(v)) + case float64: + return int64FromFloat(v) + case json.Number: + return v.Int64() + } + return 0, fmt.Errorf("unsupported %T", raw) +} + +func optionUint64(raw any) (uint64, error) { + switch v := raw.(type) { + case uint: + return uint64(v), nil + case uint8: + return uint64(v), nil + case uint16: + return uint64(v), nil + case uint32: + return uint64(v), nil + case uint64: + return v, nil + case uintptr: + return uint64(v), nil + case int: + return nonNegative(uint64(v), v < 0) + case int8: + return nonNegative(uint64(v), v < 0) + case int16: + return nonNegative(uint64(v), v < 0) + case int32: + return nonNegative(uint64(v), v < 0) + case int64: + return nonNegative(uint64(v), v < 0) + case float32: + return uint64FromFloat(float64(v)) + case float64: + return uint64FromFloat(v) + case json.Number: + return strconv.ParseUint(v.String(), 10, 64) + } + return 0, fmt.Errorf("unsupported %T", raw) +} + +func nonNegative(v uint64, negative bool) (uint64, error) { + if negative { + return 0, fmt.Errorf("negative") + } + return v, nil +} + +func int64FromUint64(v uint64) (int64, error) { + if v > math.MaxInt64 { + return 0, fmt.Errorf("overflow") + } + return int64(v), nil +} + +func int64FromFloat(v float64) (int64, error) { + if math.IsNaN(v) || math.IsInf(v, 0) || v < -maxExactFloatInteger || v > maxExactFloatInteger || math.Trunc(v) != v { + return 0, fmt.Errorf("invalid float") + } + return int64(v), nil +} + +func uint64FromFloat(v float64) (uint64, error) { + if math.IsNaN(v) || math.IsInf(v, 0) || v < 0 || v > maxExactFloatInteger || math.Trunc(v) != v { + return 0, fmt.Errorf("invalid float") + } + return uint64(v), nil +} diff --git a/api/runtime/lua/options_fuzz_test.go b/api/runtime/lua/options_fuzz_test.go new file mode 100644 index 000000000..d4f05f061 --- /dev/null +++ b/api/runtime/lua/options_fuzz_test.go @@ -0,0 +1,389 @@ +// SPDX-License-Identifier: MPL-2.0 + +package lua + +import ( + "encoding/json" + "fmt" + "math" + "strconv" + "testing" + + "github.com/wippyai/runtime/api/attrs" +) + +// optionKinds is the number of Go representations fuzzOptionValue produces. +const optionKinds = 21 + +// fuzzOptionValue builds an option value of the representation selected by +// kind from the fuzzed scalars. +func fuzzOptionValue(kind uint8, i int64, u uint64, f float64, s string) any { + switch kind % optionKinds { + case 0: + return int(i) + case 1: + return int8(i) + case 2: + return int16(i) + case 3: + return int32(i) + case 4: + return i + case 5: + return uint(u) + case 6: + return uint8(u) + case 7: + return uint16(u) + case 8: + return uint32(u) + case 9: + return u + case 10: + return float32(f) + case 11: + return f + case 12: + return json.Number(s) + case 13: + return s + case 14: + return i%2 == 0 + case 15: + return nil + case 16: + return []any{i} + case 17: + return map[string]any{"v": i} + case 18: + return []byte(s) + case 19: + return uintptr(u) + default: + return struct{}{} + } +} + +// wantTick is the reference reading of a tick_budget value: whether it is +// given, whether it is a valid integer, and its value. +func wantTick(raw any) (given, valid bool, v int64) { + switch x := raw.(type) { + case nil: + return false, false, 0 + case int: + return true, true, int64(x) + case int8: + return true, true, int64(x) + case int16: + return true, true, int64(x) + case int32: + return true, true, int64(x) + case int64: + return true, true, x + case uint: + return true, uint64(x) <= math.MaxInt64, int64(x) + case uint8: + return true, true, int64(x) + case uint16: + return true, true, int64(x) + case uint32: + return true, true, int64(x) + case uint64: + return true, x <= math.MaxInt64, int64(x) + case uintptr: + return true, uint64(x) <= math.MaxInt64, int64(x) + case float32: + return wantFloat(float64(x), true) + case float64: + return wantFloat(x, true) + case json.Number: + n, err := strconv.ParseInt(string(x), 10, 64) + return true, err == nil, n + } + return true, false, 0 +} + +// wantSteps is the reference reading of a max_steps value, which spans the +// whole uint64 range and rejects negatives. +func wantSteps(raw any) (given, valid bool, v uint64) { + switch x := raw.(type) { + case nil: + return false, false, 0 + case int: + return true, x >= 0, uint64(x) + case int8: + return true, x >= 0, uint64(x) + case int16: + return true, x >= 0, uint64(x) + case int32: + return true, x >= 0, uint64(x) + case int64: + return true, x >= 0, uint64(x) + case uint: + return true, true, uint64(x) + case uint8: + return true, true, uint64(x) + case uint16: + return true, true, uint64(x) + case uint32: + return true, true, uint64(x) + case uint64: + return true, true, x + case uintptr: + return true, true, uint64(x) + case float32: + g, ok, n := wantFloat(float64(x), false) + return g, ok, uint64(n) + case float64: + g, ok, n := wantFloat(x, false) + return g, ok, uint64(n) + case json.Number: + n, err := strconv.ParseUint(string(x), 10, 64) + return true, err == nil, n + } + return true, false, 0 +} + +func wantFloat(f float64, signed bool) (given, valid bool, v int64) { + if math.IsNaN(f) || math.IsInf(f, 0) || f != math.Trunc(f) || f > 1<<53 || f < -(1<<53) || (!signed && f < 0) { + return true, false, 0 + } + return true, true, int64(f) +} + +func checkTickBudget(t *testing.T, b ExecutionBudgets, err error, raw any) { + t.Helper() + given, valid, want := wantTick(raw) + switch { + case !given: + if err != nil || b.TickBudgetSet || b.TickBudget != 0 { + t.Fatalf("tick_budget %#v: absent option parsed to %+v, %v", raw, b, err) + } + case !valid: + if err == nil { + t.Fatalf("tick_budget %#v: expected an error, got %+v", raw, b) + } + default: + if err != nil || !b.TickBudgetSet || b.TickBudget != want { + t.Fatalf("tick_budget %#v: got %+v, %v, want %d", raw, b, err, want) + } + } +} + +func checkMaxSteps(t *testing.T, b ExecutionBudgets, err error, raw any) { + t.Helper() + given, valid, want := wantSteps(raw) + switch { + case !given: + if err != nil || b.MaxStepsSet || b.MaxSteps != 0 { + t.Fatalf("max_steps %#v: absent option parsed to %+v, %v", raw, b, err) + } + case !valid: + if err == nil { + t.Fatalf("max_steps %#v: expected an error, got %+v", raw, b) + } + default: + if err != nil || !b.MaxStepsSet || b.MaxSteps != want { + t.Fatalf("max_steps %#v: got %+v, %v, want %d", raw, b, err, want) + } + } +} + +func addOptionSeeds(f *testing.F) { + f.Helper() + ints := []int64{0, 1, -1, 2048, math.MaxInt64, math.MinInt64, 1 << 53, 1<<53 + 1} + f.Add(uint8(9), int64(0), uint64(math.MaxUint64), 0.0, "", uint8(9), int64(0), uint64(math.MaxUint64), 0.0, "18446744073709551615") + for kind := uint8(0); kind < optionKinds; kind++ { + for _, i := range ints { + f.Add(kind, i, uint64(i), float64(i), "100", kind+1, i, uint64(i), float64(i), "7") + } + f.Add(kind, int64(5), uint64(math.MaxUint64), math.NaN(), "1.5", kind, int64(-5), uint64(1)<<63, math.Inf(1), "1e3") + f.Add(kind, int64(5), uint64(5), 0.5, "", kind, int64(5), uint64(5), math.Copysign(0, -1), "99999999999999999999") + } +} + +// ExecutionBudgetsFromOptions reads tick_budget and max_steps with the +// documented semantics for every value representation. +func FuzzExecutionBudgetsFromOptions(f *testing.F) { + addOptionSeeds(f) + f.Fuzz(func(t *testing.T, tk uint8, ti int64, tu uint64, tf float64, ts string, mk uint8, mi int64, mu uint64, mf float64, ms string) { + tick := fuzzOptionValue(tk, ti, tu, tf, ts) + steps := fuzzOptionValue(mk, mi, mu, mf, ms) + + b, err := ExecutionBudgetsFromOptions(attrs.Bag{ProcessOptionTickBudget: tick}, "fuzz") + checkTickBudget(t, b, err, tick) + if b.MaxStepsSet { + t.Fatalf("max_steps reported set by tick_budget only: %+v", b) + } + + b, err = ExecutionBudgetsFromOptions(attrs.Bag{ProcessOptionMaxSteps: steps}, "fuzz") + checkMaxSteps(t, b, err, steps) + if b.TickBudgetSet { + t.Fatalf("tick_budget reported set by max_steps only: %+v", b) + } + + both, err := ExecutionBudgetsFromOptions(attrs.Bag{ProcessOptionTickBudget: tick, ProcessOptionMaxSteps: steps}, "fuzz") + tg, tv, _ := wantTick(tick) + sg, sv, _ := wantSteps(steps) + wantErr := (tg && !tv) || (sg && !sv) + if wantErr != (err != nil) { + t.Fatalf("both %#v %#v: error %v, want error %v", tick, steps, err, wantErr) + } + if err == nil { + single, _ := ExecutionBudgetsFromOptions(attrs.Bag{ProcessOptionTickBudget: tick}, "fuzz") + other, _ := ExecutionBudgetsFromOptions(attrs.Bag{ProcessOptionMaxSteps: steps}, "fuzz") + if both.TickBudget != single.TickBudget || both.TickBudgetSet != single.TickBudgetSet || + both.MaxSteps != other.MaxSteps || both.MaxStepsSet != other.MaxStepsSet { + t.Fatalf("options are not independent: both %+v tick %+v steps %+v", both, single, other) + } + } + }) +} + +// wrapOptions places the options under meta.options in one of the map +// representations an entry meta can hold. +func wrapOptions(shape uint8, options attrs.Bag) any { + switch shape % 3 { + case 0: + return map[string]any(options) + case 1: + return options + default: + return attrs.Attributes(options) + } +} + +// EntryExecutionBudgets reads only meta.options and rejects the options +// placed directly in meta or in a non-map meta.options. +func FuzzEntryExecutionBudgets(f *testing.F) { + f.Add(uint8(0), uint8(4), int64(2048), uint64(0), 0.0, "", uint8(4), int64(10), uint64(0), 0.0, "", uint8(0), uint8(0)) + f.Add(uint8(1), uint8(11), int64(0), uint64(0), 1.5, "", uint8(13), int64(0), uint64(0), 0.0, "x", uint8(1), uint8(1)) + f.Add(uint8(2), uint8(4), int64(-1), uint64(0), 0.0, "", uint8(4), int64(-1), uint64(0), 0.0, "", uint8(2), uint8(2)) + f.Fuzz(func(t *testing.T, shape uint8, tk uint8, ti int64, tu uint64, tf float64, ts string, mk uint8, mi int64, mu uint64, mf float64, ms string, direct uint8, outer uint8) { + tick := fuzzOptionValue(tk, ti, tu, tf, ts) + steps := fuzzOptionValue(mk, mi, mu, mf, ms) + options := attrs.Bag{ProcessOptionTickBudget: tick, ProcessOptionMaxSteps: steps} + + meta := attrs.Bag{entryOptionsMetaKey: wrapOptions(shape, options), "comment": "x"} + switch direct % 3 { + case 1: + meta[ProcessOptionTickBudget] = ti + case 2: + meta[ProcessOptionMaxSteps] = mi + } + got, err := EntryExecutionBudgets(meta) + if direct%3 != 0 { + if err == nil { + t.Fatalf("option placed directly in meta accepted: %+v", got) + } + return + } + want, wantErr := ExecutionBudgetsFromOptions(options, "entry") + if (err != nil) != (wantErr != nil) { + t.Fatalf("meta.options %#v: error %v, direct parse error %v", options, err, wantErr) + } + if err == nil && got != want { + t.Fatalf("meta.options %#v: got %+v, direct parse %+v", options, got, want) + } + + scalars := []any{tick, steps, "fast", int64(5), true, []any{"tick_budget"}, 1.5} + raw := scalars[int(outer)%len(scalars)] + _, err = EntryExecutionBudgets(attrs.Bag{entryOptionsMetaKey: raw}) + switch raw.(type) { + case map[string]any: + // The only map among the candidates holds the unknown key "v". + if err == nil { + t.Fatal("meta.options with an unknown field accepted") + } + case nil: + if err != nil { + t.Fatalf("null meta.options rejected: %v", err) + } + default: + if err == nil { + t.Fatalf("meta.options %T accepted", raw) + } + } + }) +} + +// Parsing an entry and a spawn bag then overriding follows the precedence +// rule: an option the spawn gives replaces the entry's, even an explicit zero, +// and an option it omits leaves the entry's value. +func FuzzExecutionBudgetsOverride(f *testing.F) { + f.Add(int64(100), int64(5), uint8(0), int64(-1), int64(0), uint8(1), uint8(1)) + f.Add(int64(0), int64(0), uint8(2), int64(7), int64(7), uint8(0), uint8(2)) + f.Add(int64(-1), int64(1<<40), uint8(3), int64(0), int64(0), uint8(3), uint8(3)) + f.Fuzz(func(t *testing.T, et, es int64, em uint8, st, ss int64, sm uint8, _ uint8) { + entryOpts := attrs.Bag{} + spawnOpts := attrs.Bag{} + if em&1 != 0 { + entryOpts[ProcessOptionTickBudget] = et + } + if em&2 != 0 { + entryOpts[ProcessOptionMaxSteps] = es + } + if sm&1 != 0 { + spawnOpts[ProcessOptionTickBudget] = st + } + if sm&2 != 0 { + spawnOpts[ProcessOptionMaxSteps] = ss + } + entry, entryErr := EntryExecutionBudgets(attrs.Bag{entryOptionsMetaKey: entryOpts}) + spawn, spawnErr := ExecutionBudgetsFromOptions(spawnOpts, "spawn") + if entryErr != nil || spawnErr != nil { + // Only max_steps below zero is invalid among int64 inputs. + if (entryErr != nil) != (em&2 != 0 && es < 0) || (spawnErr != nil) != (sm&2 != 0 && ss < 0) { + t.Fatalf("unexpected errors: entry %v spawn %v", entryErr, spawnErr) + } + return + } + got := entry.Override(spawn) + + wantTick, wantTickSet := entry.TickBudget, entry.TickBudgetSet + if sm&1 != 0 { + wantTick, wantTickSet = st, true + } + wantSteps, wantStepsSet := entry.MaxSteps, entry.MaxStepsSet + if sm&2 != 0 { + wantSteps, wantStepsSet = uint64(ss), true + } + if got.TickBudget != wantTick || got.TickBudgetSet != wantTickSet || got.MaxSteps != wantSteps || got.MaxStepsSet != wantStepsSet { + t.Fatalf("override(%+v, %+v) = %+v", entry, spawn, got) + } + if (ExecutionBudgets{}).Override(spawn) != spawn { + t.Fatalf("zero budgets overridden by %+v changed it", spawn) + } + if entry.Override(ExecutionBudgets{}) != entry { + t.Fatalf("empty override changed %+v", entry) + } + }) +} + +// A process entry accepts the v2 process option keys and default_host in +// meta.options, and rejects every other key by name. +func FuzzEntryOptionKeys(f *testing.F) { + for _, k := range []string{"max_step", "tick_budgets", "default_host", "network", "hot_policy_id", "mailbox_capacity", "", "TICK_BUDGET", "pool"} { + f.Add(k, int64(3)) + } + known := map[string]bool{ + "mailbox_capacity": true, "memory_limit_bytes": true, "heap_reserve_bytes": true, + "hot_policy_id": true, "network": true, "default_host": true, + } + f.Fuzz(func(t *testing.T, key string, v int64) { + if key == ProcessOptionTickBudget || key == ProcessOptionMaxSteps { + t.Skip() + } + _, err := EntryExecutionBudgets(attrs.Bag{entryOptionsMetaKey: attrs.Bag{key: v}}) + if known[key] { + if err != nil { + t.Fatalf("known option %q rejected: %v", key, err) + } + return + } + want := fmt.Sprintf("process entry meta.options has unknown field %q", key) + if err == nil || err.Error() != want { + t.Fatalf("unknown option %q: got %v, want %q", key, err, want) + } + }) +} diff --git a/api/runtime/lua/options_test.go b/api/runtime/lua/options_test.go new file mode 100644 index 000000000..c1b292ce3 --- /dev/null +++ b/api/runtime/lua/options_test.go @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: MPL-2.0 + +package lua + +import ( + "encoding/json" + "testing" + + "github.com/wippyai/runtime/api/attrs" +) + +func TestEntryExecutionBudgetsFromMetaOptions(t *testing.T) { + b, err := EntryExecutionBudgets(attrs.Bag{"options": map[string]any{"tick_budget": 2048, "max_steps": float64(10)}}) + if err != nil { + t.Fatal(err) + } + if !b.TickBudgetSet || b.TickBudget != 2048 || !b.MaxStepsSet || b.MaxSteps != 10 { + t.Fatalf("unexpected budgets %+v", b) + } + + none, err := EntryExecutionBudgets(attrs.Bag{"comment": "x"}) + if err != nil || none.TickBudgetSet || none.MaxStepsSet { + t.Fatalf("expected no budgets, got %+v %v", none, err) + } +} + +func TestEntryExecutionBudgetsRejectsDirectMetaKeys(t *testing.T) { + for _, key := range []string{ProcessOptionTickBudget, ProcessOptionMaxSteps} { + if _, err := EntryExecutionBudgets(attrs.Bag{key: 10}); err == nil { + t.Fatalf("%s directly under meta must be rejected", key) + } + } +} + +func TestExecutionBudgetsFromOptionsValues(t *testing.T) { + valid := []struct { + raw any + want int64 + }{ + {int(-1), -1}, + {int64(512), 512}, + {uint32(7), 7}, + {float64(4096), 4096}, + {json.Number("100"), 100}, + } + for _, tc := range valid { + b, err := ExecutionBudgetsFromOptions(attrs.Bag{ProcessOptionTickBudget: tc.raw}, "test") + if err != nil || b.TickBudget != tc.want || !b.TickBudgetSet { + t.Fatalf("tick_budget %v: got %+v %v", tc.raw, b, err) + } + } + invalid := []attrs.Bag{ + {ProcessOptionTickBudget: "512"}, + {ProcessOptionTickBudget: 1.5}, + {ProcessOptionTickBudget: float64(1 << 60)}, + {ProcessOptionMaxSteps: -1}, + {ProcessOptionMaxSteps: true}, + } + for _, opts := range invalid { + if _, err := ExecutionBudgetsFromOptions(opts, "test"); err == nil { + t.Fatalf("expected %v to be rejected", opts) + } + } +} + +func TestExecutionBudgetsOverride(t *testing.T) { + entry := ExecutionBudgets{TickBudget: 100, TickBudgetSet: true, MaxSteps: 5, MaxStepsSet: true} + got := entry.Override(ExecutionBudgets{TickBudget: -1, TickBudgetSet: true}) + if got.TickBudget != -1 || got.MaxSteps != 5 || !got.MaxStepsSet { + t.Fatalf("unexpected override %+v", got) + } +} + +func TestEntryExecutionBudgetsRejectsNonMapOptions(t *testing.T) { + for _, raw := range []any{"fast", []any{"tick_budget"}, 5} { + if _, err := EntryExecutionBudgets(attrs.Bag{"options": raw}); err == nil { + t.Fatalf("meta.options %T must be rejected", raw) + } + } + if b, err := EntryExecutionBudgets(attrs.Bag{"options": nil}); err != nil || b.TickBudgetSet || b.MaxStepsSet { + t.Fatalf("null meta.options holds no options, got %+v %v", b, err) + } +} + +func TestExecutionBudgetsFromNilOptionsIsEmpty(t *testing.T) { + b, err := ExecutionBudgetsFromOptions(nil, "test") + if err != nil || b.TickBudgetSet || b.MaxStepsSet { + t.Fatalf("expected empty budgets, got %+v %v", b, err) + } +} + +func TestEntryExecutionBudgetsRejectsUnknownOptionFields(t *testing.T) { + _, err := EntryExecutionBudgets(attrs.Bag{"options": attrs.Bag{"max_step": 5}}) + if err == nil || err.Error() != `process entry meta.options has unknown field "max_step"` { + t.Fatalf("unexpected error %v", err) + } +} + +func TestEntryExecutionBudgetsAcceptsProcessEntryOptionKeys(t *testing.T) { + options := attrs.Bag{} + for _, key := range []string{"mailbox_capacity", "memory_limit_bytes", "heap_reserve_bytes", "hot_policy_id", "network", "default_host"} { + options[key] = "x" + } + if _, err := EntryExecutionBudgets(attrs.Bag{"options": options}); err != nil { + t.Fatal(err) + } +} + +func TestExecutionBudgetsMaxStepsSpansUint64(t *testing.T) { + const max = ^uint64(0) + for _, raw := range []any{max, uint(max), uintptr(max), json.Number("18446744073709551615")} { + b, err := ExecutionBudgetsFromOptions(attrs.Bag{ProcessOptionMaxSteps: raw}, "test") + if err != nil || !b.MaxStepsSet || b.MaxSteps != max { + t.Fatalf("max_steps %#v: got %+v %v", raw, b, err) + } + } + if _, err := ExecutionBudgetsFromOptions(attrs.Bag{ProcessOptionTickBudget: uint64(1) << 63}, "test"); err == nil { + t.Fatal("tick_budget above int64 must be rejected") + } + if b, err := ExecutionBudgetsFromOptions(attrs.Bag{ProcessOptionTickBudget: uintptr(9)}, "test"); err != nil || b.TickBudget != 9 { + t.Fatalf("uintptr tick_budget: %+v %v", b, err) + } +} diff --git a/go.mod b/go.mod index ca0ffc464..37b6c3f9d 100644 --- a/go.mod +++ b/go.mod @@ -69,7 +69,7 @@ require ( github.com/tree-sitter/tree-sitter-php v0.25.0 github.com/tree-sitter/tree-sitter-python v0.25.0 github.com/tree-sitter/tree-sitter-typescript v0.23.2 - github.com/wippyai/go-lua v1.6.2 + github.com/wippyai/go-lua v1.6.4-0.20261003205127-c564eb385ccd github.com/wippyai/module-registry-proto-go v0.0.2-0.20260908140534-f6e2910c835f github.com/wippyai/tree-sitter-markdown v0.0.3 github.com/wippyai/tree-sitter-sql v0.0.4 diff --git a/go.sum b/go.sum index ceefeca46..b24a71513 100644 --- a/go.sum +++ b/go.sum @@ -544,6 +544,22 @@ github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zd github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/wippyai/go-lua v1.6.2 h1:SJRjjW0XjDTVbFmPtjs3NHoHz1UgNdXMFJlCPg6aTQM= github.com/wippyai/go-lua v1.6.2/go.mod h1:cwD+390gJgx9cs+Zamby/lch5csqPV9RN1aDEfx/27M= +github.com/wippyai/go-lua v1.6.4-0.20261002221722-a0cd38afeb7d h1:ZJjb1KcC83wKn4ndVPXJ1r2OJoZZneIuKMPxwwXRJYE= +github.com/wippyai/go-lua v1.6.4-0.20261002221722-a0cd38afeb7d/go.mod h1:cwD+390gJgx9cs+Zamby/lch5csqPV9RN1aDEfx/27M= +github.com/wippyai/go-lua v1.6.4-0.20261003005521-47a27bd56cfd h1:vmuy4HscyBNOQs9DI75/yEkba4kQWhFkIv5YZkU3Dd8= +github.com/wippyai/go-lua v1.6.4-0.20261003005521-47a27bd56cfd/go.mod h1:cwD+390gJgx9cs+Zamby/lch5csqPV9RN1aDEfx/27M= +github.com/wippyai/go-lua v1.6.4-0.20261003012915-c14848bbdb76 h1:jTTXVVQ9q3HK2SUnFnzQj2SCInmKFTKTlE4XDdZ3WbQ= +github.com/wippyai/go-lua v1.6.4-0.20261003012915-c14848bbdb76/go.mod h1:cwD+390gJgx9cs+Zamby/lch5csqPV9RN1aDEfx/27M= +github.com/wippyai/go-lua v1.6.4-0.20261003025238-2c373075d09d h1:NpoQ7a1P/B1u/hVLvV2l4IR9ESMsv0NWziLXmSzscIk= +github.com/wippyai/go-lua v1.6.4-0.20261003025238-2c373075d09d/go.mod h1:cwD+390gJgx9cs+Zamby/lch5csqPV9RN1aDEfx/27M= +github.com/wippyai/go-lua v1.6.4-0.20261003135019-e6c9f2e920e8 h1:cPF3sBY4gbJi5wDrecxyjkdjtgn4qAFAtnuPsbyO/Kw= +github.com/wippyai/go-lua v1.6.4-0.20261003135019-e6c9f2e920e8/go.mod h1:cwD+390gJgx9cs+Zamby/lch5csqPV9RN1aDEfx/27M= +github.com/wippyai/go-lua v1.6.4-0.20261003145456-1a12b5df98d5 h1:sJpAq4jGwV5q/co5qPnfi981X9dPkDKh8bdPGf068Aw= +github.com/wippyai/go-lua v1.6.4-0.20261003145456-1a12b5df98d5/go.mod h1:cwD+390gJgx9cs+Zamby/lch5csqPV9RN1aDEfx/27M= +github.com/wippyai/go-lua v1.6.4-0.20261003172345-af635159323f h1:HBoZvymdzm4yklVwPAJlJIREypgRRcUeMNvdoKtIVdM= +github.com/wippyai/go-lua v1.6.4-0.20261003172345-af635159323f/go.mod h1:cwD+390gJgx9cs+Zamby/lch5csqPV9RN1aDEfx/27M= +github.com/wippyai/go-lua v1.6.4-0.20261003205127-c564eb385ccd h1:Y6q7oiqgg7pKiV3A026Foiaqi/tvuvxdYn3sGZYGrgE= +github.com/wippyai/go-lua v1.6.4-0.20261003205127-c564eb385ccd/go.mod h1:cwD+390gJgx9cs+Zamby/lch5csqPV9RN1aDEfx/27M= github.com/wippyai/module-registry-proto-go v0.0.2-0.20260908140534-f6e2910c835f h1:AoKS058roqnsHXaJ8Gww7ODz14qJqOK5QBeoALvcq6I= github.com/wippyai/module-registry-proto-go v0.0.2-0.20260908140534-f6e2910c835f/go.mod h1:p4ihYQKRQuqRVLxaH1YL0IEnkz4zKdyfVADOwvA5Tno= github.com/wippyai/tree-sitter-markdown v0.0.3 h1:u6e53+hzPSS848Xhm+I48SwtvWQHrC21wDrogdwCwuc= diff --git a/runtime/lua/component/process/manager.go b/runtime/lua/component/process/manager.go index 3d21fa0d9..0a0653e35 100644 --- a/runtime/lua/component/process/manager.go +++ b/runtime/lua/component/process/manager.go @@ -28,6 +28,7 @@ type configEntry struct { bytecode *api.BytecodeProcessConfig security *security.Config method string + budgets api.ExecutionBudgets } // Manager handles both source and bytecode Lua process components. @@ -117,7 +118,7 @@ func (m *Manager) Invalidate(ctx context.Context, ids []registry.ID) error { } } - if err := m.registerFactory(ctx, id, cfg.method, cfg.security); err != nil { + if err := m.registerFactory(ctx, id, cfg); err != nil { m.log.Error("failed to invalidate process", zap.Error(err)) errs = append(errs, err) continue @@ -156,6 +157,10 @@ func (m *Manager) addSource(ctx context.Context, entry registry.Entry) error { if err != nil { return runtimelua.NewUnpackConfigError("process", err) } + budgets, err := api.EntryExecutionBudgets(cfg.Meta) + if err != nil { + return runtimelua.NewUnpackConfigError("process", err) + } node := code.Node{ ID: entry.ID, @@ -168,9 +173,10 @@ func (m *Manager) addSource(ctx context.Context, entry registry.Entry) error { return runtimelua.NewAddNodeError("process", err) } - m.configs.Store(entry.ID, &configEntry{method: cfg.Method, source: cfg, security: cfg.Security}) + stored := &configEntry{method: cfg.Method, source: cfg, security: cfg.Security, budgets: budgets} + m.configs.Store(entry.ID, stored) - if err := m.registerFactory(ctx, entry.ID, cfg.Method, cfg.Security); err != nil { + if err := m.registerFactory(ctx, entry.ID, stored); err != nil { _ = m.code.DeleteNode(ctx, entry.ID) m.configs.Delete(entry.ID) return runtimelua.NewRegisterFactoryError(err) @@ -186,6 +192,10 @@ func (m *Manager) addBytecode(ctx context.Context, entry registry.Entry) error { if err != nil { return runtimelua.NewUnpackConfigError("process", err) } + budgets, err := api.EntryExecutionBudgets(cfg.Meta) + if err != nil { + return runtimelua.NewUnpackConfigError("process", err) + } proto, err := component.LoadAndVerifyBytecode(m.fsRegistry, cfg.FS, cfg.Path, cfg.Hash) if err != nil { @@ -202,9 +212,10 @@ func (m *Manager) addBytecode(ctx context.Context, entry registry.Entry) error { return runtimelua.NewAddNodeError("process", err) } - m.configs.Store(entry.ID, &configEntry{method: cfg.Method, bytecode: cfg, security: cfg.Security}) + stored := &configEntry{method: cfg.Method, bytecode: cfg, security: cfg.Security, budgets: budgets} + m.configs.Store(entry.ID, stored) - if err := m.registerFactory(ctx, entry.ID, cfg.Method, cfg.Security); err != nil { + if err := m.registerFactory(ctx, entry.ID, stored); err != nil { _ = m.code.DeleteNode(ctx, entry.ID) m.configs.Delete(entry.ID) return runtimelua.NewRegisterFactoryError(err) @@ -224,6 +235,10 @@ func (m *Manager) updateSource(ctx context.Context, entry registry.Entry) error if err != nil { return runtimelua.NewUnpackConfigError("process", err) } + budgets, err := api.EntryExecutionBudgets(cfg.Meta) + if err != nil { + return runtimelua.NewUnpackConfigError("process", err) + } node := code.Node{ ID: entry.ID, @@ -236,9 +251,10 @@ func (m *Manager) updateSource(ctx context.Context, entry registry.Entry) error return runtimelua.NewUpdateNodeError("process", err) } - m.configs.Store(entry.ID, &configEntry{method: cfg.Method, source: cfg, security: cfg.Security}) + stored := &configEntry{method: cfg.Method, source: cfg, security: cfg.Security, budgets: budgets} + m.configs.Store(entry.ID, stored) - if err := m.registerFactory(ctx, entry.ID, cfg.Method, cfg.Security); err != nil { + if err := m.registerFactory(ctx, entry.ID, stored); err != nil { return runtimelua.NewUpdateFactoryError(err) } @@ -252,6 +268,10 @@ func (m *Manager) updateBytecode(ctx context.Context, entry registry.Entry) erro if err != nil { return runtimelua.NewUnpackConfigError("process", err) } + budgets, err := api.EntryExecutionBudgets(cfg.Meta) + if err != nil { + return runtimelua.NewUnpackConfigError("process", err) + } proto, err := component.LoadAndVerifyBytecode(m.fsRegistry, cfg.FS, cfg.Path, cfg.Hash) if err != nil { @@ -268,9 +288,10 @@ func (m *Manager) updateBytecode(ctx context.Context, entry registry.Entry) erro return runtimelua.NewUpdateNodeError("process", err) } - m.configs.Store(entry.ID, &configEntry{method: cfg.Method, bytecode: cfg, security: cfg.Security}) + stored := &configEntry{method: cfg.Method, bytecode: cfg, security: cfg.Security, budgets: budgets} + m.configs.Store(entry.ID, stored) - if err := m.registerFactory(ctx, entry.ID, cfg.Method, cfg.Security); err != nil { + if err := m.registerFactory(ctx, entry.ID, stored); err != nil { return runtimelua.NewUpdateFactoryError(err) } @@ -279,13 +300,16 @@ func (m *Manager) updateBytecode(ctx context.Context, entry registry.Entry) erro } // registerFactory registers a process factory with the factory registry and waits for confirmation. -func (m *Manager) registerFactory(ctx context.Context, id registry.ID, method string, sec *security.Config) error { +func (m *Manager) registerFactory(ctx context.Context, id registry.ID, cfg *configEntry) error { // Create factory using ProcessFactory - factoryFn, err := m.factory.CreateFactory(id, engine.WithModules(component.ExecutableAmbientModules()...)) + factoryFn, err := m.factory.CreateFactory(id, + engine.WithModules(component.ExecutableAmbientModules()...), + engine.WithExecutionBudgets(cfg.budgets)) if err != nil { return err // Already has compile context from code.Manager } + method := cfg.method if method == "" { method = "main" } @@ -311,7 +335,7 @@ func (m *Manager) registerFactory(ctx context.Context, id registry.ID, method st Factory: factoryFn, Meta: process.Meta{ Method: method, - Security: sec, + Security: cfg.security, }, }, }) diff --git a/runtime/lua/component/process/manager_test.go b/runtime/lua/component/process/manager_test.go index 46a7a050e..0b5a0ef90 100644 --- a/runtime/lua/component/process/manager_test.go +++ b/runtime/lua/component/process/manager_test.go @@ -216,7 +216,7 @@ func TestManager_registerFactory_PreparesBeforeSend(t *testing.T) { ctx := event.WithAwaitService(ctxapi.NewRootContext(), awaitSvc) securityConfig := &security.Config{Actor: security.Actor{ID: "app.test:process"}} - err := manager.registerFactory(ctx, registry.NewID("app.test", "process"), "main", securityConfig) + err := manager.registerFactory(ctx, registry.NewID("app.test", "process"), &configEntry{method: "main", security: securityConfig}) require.NoError(t, err) assert.False(t, sendBeforePrepare, "factory register was sent before await prepare") require.Len(t, bus.events, 1) diff --git a/runtime/lua/engine/bench_test.go b/runtime/lua/engine/bench_test.go index 1c9a9afdf..18925557b 100644 --- a/runtime/lua/engine/bench_test.go +++ b/runtime/lua/engine/bench_test.go @@ -955,6 +955,8 @@ func (be *benchExecutor) Execute(ctx context.Context, p pidapi.PID, proc process _, err := be.sched.Submit(ctx, p, proc, method, input) if err != nil { + // Failed admission leaves ownership with the caller, as in service/host. + proc.Close() be.mu.Lock() delete(be.pending, p.UniqID) be.mu.Unlock() diff --git a/runtime/lua/engine/bootstrap_test.go b/runtime/lua/engine/bootstrap_test.go new file mode 100644 index 000000000..53f6fb02f --- /dev/null +++ b/runtime/lua/engine/bootstrap_test.go @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: MPL-2.0 + +package engine + +import ( + "context" + "testing" + + lua "github.com/wippyai/go-lua" + ctxapi "github.com/wippyai/runtime/api/context" + "github.com/wippyai/runtime/api/payload" + "github.com/wippyai/runtime/api/process" +) + +// The module bootstrap does not use globals the guest can change. +func TestProcessModuleBootstrapIgnoresShadowedGlobals(t *testing.T) { + proc := mustNewProcess(t, WithScript(` + select = nil + pcall = nil + unpack = nil + table = nil + return { main = function(a, b) return a + b end } + `, "module.lua")) + t.Cleanup(proc.Close) + ctx, _ := ctxapi.OpenFrameContext(context.Background()) + args := payload.Payloads{&testPayload{val: lua.LNumber(10)}, &testPayload{val: lua.LNumber(20)}} + if err := proc.Init(ctx, "main", args); err != nil { + t.Fatal(err) + } + var output process.StepOutput + if err := proc.Step(nil, &output); err != nil { + t.Fatal(err) + } + if v, _ := output.Result().Data().(lua.LValue); lua.LVAsNumber(v) != 30 { + t.Fatalf("expected 30, got %v", output.Result().Data()) + } +} diff --git a/runtime/lua/engine/budget_flush_test.go b/runtime/lua/engine/budget_flush_test.go new file mode 100644 index 000000000..e4233d217 --- /dev/null +++ b/runtime/lua/engine/budget_flush_test.go @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: MPL-2.0 + +package engine + +import ( + "fmt" + "testing" + + lua "github.com/wippyai/go-lua" + ctxapi "github.com/wippyai/runtime/api/context" + "github.com/wippyai/runtime/api/payload" + "github.com/wippyai/runtime/api/process" + "github.com/wippyai/runtime/api/relay" + luaapi "github.com/wippyai/runtime/api/runtime/lua" +) + +// A message held back by a full channel is delivered to the receiver that +// parks on it even when that receive uses up the step's last tick: the +// process never reports idle while a queued message matches a parked receiver. +func TestProcessExhaustedBudgetDeliversQueuedMessage(t *testing.T) { + for budget := int64(1); budget <= 40; budget++ { + t.Run(fmt.Sprintf("budget=%d", budget), func(t *testing.T) { + proto, err := lua.CompileString(` + local ch = channel.new(1) + subscribe("inbox", ch) + local a = ch:receive() + local b = ch:receive() + return b + `, "receive.lua") + if err != nil { + t.Fatal(err) + } + proc := mustNewProcess(t, WithProto(proto), WithProcessExecutionBudgets(luaapi.ExecutionBudgets{TickBudget: budget, TickBudgetSet: true})) + proc.EnablePreemption() + t.Cleanup(proc.Close) + ctx, _ := ctxapi.OpenFrameContext(ctxapi.NewRootContext()) + ctx = payload.WithTranscoder(ctx, createInboxTestTranscoder()) + if err := proc.Init(ctx, "", nil); err != nil { + t.Fatal(err) + } + LoadModuleDef(proc.State(), ChannelModule) + loadPubSubGlobals(proc.State()) + + var output process.StepOutput + events := []process.Event{{ + Type: process.EventMessage, + Data: &relay.Package{Messages: []*relay.Message{ + {Topic: "inbox", Payloads: payload.Payloads{payload.NewPayload(lua.LString("first"), payload.Lua)}}, + {Topic: "inbox", Payloads: payload.Payloads{payload.NewPayload(lua.LString("second"), payload.Lua)}}, + }}, + }} + for i := 0; i < 200; i++ { + output.Reset() + if err := proc.Step(events, &output); err != nil { + t.Fatal(err) + } + events = nil + switch output.Status() { + case process.StepDone: + return + case process.StepIdle: + t.Fatalf("step %d: idle with a queued message for the parked receiver", i) + } + } + t.Fatal("process did not complete") + }) + } +} diff --git a/runtime/lua/engine/chaos_test.go b/runtime/lua/engine/chaos_test.go new file mode 100644 index 000000000..4cf0d8a31 --- /dev/null +++ b/runtime/lua/engine/chaos_test.go @@ -0,0 +1,732 @@ +// SPDX-License-Identifier: MPL-2.0 + +package engine + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "math" + "math/rand" + "os" + goruntime "runtime" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + lua "github.com/wippyai/go-lua" + "github.com/wippyai/runtime/api/attrs" + ctxapi "github.com/wippyai/runtime/api/context" + "github.com/wippyai/runtime/api/dispatcher" + "github.com/wippyai/runtime/api/payload" + pidapi "github.com/wippyai/runtime/api/pid" + "github.com/wippyai/runtime/api/process" + "github.com/wippyai/runtime/api/registry" + "github.com/wippyai/runtime/api/relay" + "github.com/wippyai/runtime/api/runtime" + luaapi "github.com/wippyai/runtime/api/runtime/lua" + scheduler "github.com/wippyai/runtime/system/scheduler/actor" +) + +// Engine chaos: real Lua actors under the actor scheduler with random +// tick_budget and max_steps, driven by a seed. Environment: +// +// WIPPY_LUA_CHAOS_SEED first seed (default 1) +// WIPPY_LUA_CHAOS_RUNS number of consecutive seeds (default 3, 100 with WIPPY_CHAOS_LONG) +// WIPPY_LUA_CHAOS_PROCS actors per run (default 24, 150 with WIPPY_CHAOS_LONG) +func luaChaosEnvInt(name string, short, long int) int { + def := short + if os.Getenv("WIPPY_CHAOS_LONG") != "" { + def = long + } + if v := os.Getenv(name); v != "" { + if n, err := strconv.Atoi(v); err == nil { + return n + } + } + return def +} + +var errLuaChaosStart = errors.New("lua chaos: start rejected") + +type luaStage struct { + body string // Lua statements that leave the stage result in `result` + expected string // value of `result` + // entry budgets of the incarnation + entry luaapi.ExecutionBudgets + // delayed completion left outstanding by the incarnation before it upgrades + staleMs int + // upgrade requests an in-place upgrade after the body + upgrade bool +} + +type luaRec struct { + rng *rand.Rand + cancel context.CancelFunc + spawn attrs.Bag + ready chan struct{} + result *runtime.Result + pid pidapi.PID + stages []luaStage + spawnBudget luaapi.ExecutionBudgets + id int + resultMu sync.Mutex + starts atomic.Int32 + upgrades atomic.Int32 + submitted atomic.Bool + gaveUp atomic.Bool + killed atomic.Bool + running atomic.Int32 + completes atomic.Int32 + blocker bool + failing bool + wantsMsgs bool + wantsWake bool +} + +// limited reports whether any incarnation of the actor can be stopped by a +// max_steps limit. +func (r *luaRec) limited() bool { + if r.spawnBudget.MaxSteps > 0 { + return true + } + if r.spawnBudget.MaxStepsSet { + return false + } + for _, s := range r.stages { + if s.entry.MaxSteps > 0 { + return true + } + } + return false +} + +// maxStepsOf is the effective max_steps of an incarnation. +func (r *luaRec) maxStepsOf(stage luaStage) uint64 { + return stage.entry.Override(r.spawnBudget).MaxSteps +} + +type luaChaos struct { + sched *scheduler.Scheduler + appCtx context.Context + recs []*luaRec + incs []*chaosLua + violations []string + seed int64 + mu sync.Mutex + stopped atomic.Bool +} + +func (w *luaChaos) violatef(format string, args ...any) { + w.mu.Lock() + defer w.mu.Unlock() + if len(w.violations) < 40 { + w.violations = append(w.violations, fmt.Sprintf(format, args...)) + } +} + +// chaosLua observes one incarnation's Lua process. +type chaosLua struct { + *Process + w *luaChaos + rec *luaRec + inc int + limit uint64 + inits atomic.Int32 + closed atomic.Int32 + running atomic.Int32 + limited atomic.Bool +} + +func (p *chaosLua) Init(ctx context.Context, method string, input payload.Payloads) error { + p.inits.Add(1) + return p.Process.Init(ctx, method, input) +} + +func (p *chaosLua) Step(events []process.Event, out *process.StepOutput) error { + if !p.running.CompareAndSwap(0, 1) { + p.w.violatef("rec %d inc %d: concurrent steps of one incarnation", p.rec.id, p.inc) + } + defer p.running.Store(0) + if p.rec.running.Add(1) != 1 { + p.w.violatef("rec %d: steps of two incarnations overlap", p.rec.id) + } + defer p.rec.running.Add(-1) + if p.rec.completes.Load() != 0 { + p.w.violatef("rec %d inc %d: step after completion", p.rec.id, p.inc) + } + if p.closed.Load() != 0 { + p.w.violatef("rec %d inc %d: step after close", p.rec.id, p.inc) + } + if p.limited.Load() { + p.w.violatef("rec %d inc %d: step after the step limit failed it", p.rec.id, p.inc) + } + + err := p.Process.Step(events, out) + switch { + case errors.Is(err, process.ErrStepLimitExceeded): + p.limited.Store(true) + if p.limit == 0 { + p.w.violatef("rec %d inc %d: step limit error without max_steps", p.rec.id, p.inc) + } + case p.limit > 0 && p.StepsUsed() > p.limit: + p.w.violatef("rec %d inc %d: %d steps used with max_steps %d (err %v)", p.rec.id, p.inc, p.StepsUsed(), p.limit, err) + } + return err +} + +func (p *chaosLua) Close() { + if p.closed.Add(1) != 1 { + p.w.violatef("rec %d inc %d: closed more than once", p.rec.id, p.inc) + return + } + if p.running.Load() != 0 { + p.w.violatef("rec %d inc %d: closed during a step", p.rec.id, p.inc) + } + p.Process.Close() +} + +type luaChaosFactory struct{ w *luaChaos } + +func (f luaChaosFactory) Create(id registry.ID) (process.Process, *process.Meta, error) { + n, err := strconv.Atoi(id.Name) + if err != nil || n < 0 || n >= len(f.w.recs) { + return nil, nil, fmt.Errorf("unknown chaos source %v", id) + } + rec := f.w.recs[n] + inc := int(rec.upgrades.Add(1)) + if inc >= len(rec.stages) { + f.w.violatef("rec %d: upgrade to incarnation %d but there are %d", rec.id, inc, len(rec.stages)) + return nil, nil, errLuaChaosStart + } + p, err := f.w.newProc(rec, inc) + if err != nil { + return nil, nil, err + } + return p, &process.Meta{}, nil +} + +func (w *luaChaos) newProc(rec *luaRec, inc int) (*chaosLua, error) { + stage := rec.stages[inc] + src := w.script(rec, inc) + upgradeReq := &UpgradeRequest{Source: registry.NewID("chaos", strconv.Itoa(rec.id))} + proc, err := NewProcess( + WithScript(src, fmt.Sprintf("chaos-%d-%d.lua", rec.id, inc)), + WithProcessExecutionBudgets(stage.entry), + WithModuleBinder(LoadCoreModules), + WithModuleBinder(bindTestYield), + WithModuleBinder(wrapBinder(func(l *lua.LState) { l.SetGlobal("upgrade_request", upgradeReq) })), + ) + if err != nil { + return nil, err + } + p := &chaosLua{Process: proc, w: w, rec: rec, inc: inc, limit: rec.maxStepsOf(stage)} + w.mu.Lock() + w.incs = append(w.incs, p) + w.mu.Unlock() + return p, nil +} + +func (w *luaChaos) script(rec *luaRec, inc int) string { + stage := rec.stages[inc] + var b strings.Builder + b.WriteString(stage.body) + b.WriteString("\n") + if stage.upgrade { + if stage.staleMs > 0 { + // A message wakes the actor while its spawned coroutine's yield is + // still outstanding; the actor then upgrades. + fmt.Fprintf(&b, "local wake = channel.new(1)\nsubscribe('wake', wake)\ncoroutine.spawn(function() test_yield(%d) end)\ncoroutine.yield()\nwake:receive()\n", stage.staleMs) + } + b.WriteString("coroutine.yield(upgrade_request)\n") + } + fmt.Fprintf(&b, "return \"gen%d:\" .. tostring(result)\n", inc) + return b.String() +} + +// luaChaosHandler completes a test_yield with its duration in milliseconds, +// after that duration: an actor that receives another value was resumed with +// a completion of a different yield. +func luaChaosHandler() dispatcher.Handler { + return dispatcher.HandlerFunc(func(_ context.Context, cmd dispatcher.Command, tag uint64, r dispatcher.ResultReceiver) error { + d := cmd.(testYieldCmd).Duration + ms := d.Milliseconds() + if d == 0 { + r.CompleteYield(tag, ms, nil) + return nil + } + time.AfterFunc(d, func() { r.CompleteYield(tag, ms, nil) }) + return nil + }) +} + +func luaTriangle(n int) int { return n * (n + 1) / 2 } + +// luaStageBody builds the body of a stage and its expected result. +func luaStageBody(rng *rand.Rand, rec *luaRec) (body, expected string) { + switch rng.Intn(6) { + case 0: + n := 100 + rng.Intn(2000) + return fmt.Sprintf("local s = 0\nfor i = 1, %d do s = s + i end\nlocal result = s", n), strconv.Itoa(luaTriangle(n)) + case 1: + k, n, y := 2+rng.Intn(5), 50+rng.Intn(400), 1+rng.Intn(40) + return fmt.Sprintf(` +local ch = channel.new(%[1]d) +for c = 1, %[1]d do + coroutine.spawn(function() + local s = 0 + for i = 1, %[2]d do + s = s + i + if i %% %[3]d == 0 then coroutine.yield() end + end + ch:send(s) + end) +end +local total = 0 +for c = 1, %[1]d do total = total + ch:receive() end +local result = total`, k, n, y), strconv.Itoa(k * luaTriangle(n)) + case 2: + r := 1 + rng.Intn(4) + var sb strings.Builder + sum := 0 + sb.WriteString("local acc = 0\n") + for i := 0; i < r; i++ { + ms := rng.Intn(5) + fmt.Fprintf(&sb, "do local r = test_yield(%d) if tonumber(r) ~= %d then error('wrong completion ' .. tostring(r) .. ' for %d') end acc = acc + tonumber(r) end\n", ms, ms, ms) + sum += ms + } + sb.WriteString("local result = acc") + return sb.String(), strconv.Itoa(sum) + case 3: + k := 2 + rng.Intn(4) + var sum int + for c := 1; c <= k; c++ { + sum += c + } + return fmt.Sprintf(` +local ch = channel.new(%[1]d) +for c = 1, %[1]d do + coroutine.spawn(function() + local spin = 0 + for i = 1, 200 do spin = spin + i end + local r = test_yield(c) + if tonumber(r) ~= c then error('wrong completion ' .. tostring(r) .. ' for ' .. c) end + ch:send(c) + end) +end +local total = 0 +for c = 1, %[1]d do total = total + ch:receive() end +local result = total`, k), strconv.Itoa(sum) + case 4: + m := 3 + rng.Intn(40) + rec.wantsMsgs = true + return fmt.Sprintf(` +local ch = channel.new(64) +subscribe("msg", ch) +local n = 0 +while n < %d do + ch:receive() + n = n + 1 + for i = 1, 50 do n = n + 0 end +end +local result = n`, m), strconv.Itoa(m) + default: + n := 20 + rng.Intn(500) + return fmt.Sprintf(` +local function fib(n) if n < 2 then return n end return fib(n - 1) + fib(n - 2) end +local result = fib(%d)`, 5+n%12), strconv.Itoa(fibRef(5 + n%12)) + } +} + +func fibRef(n int) int { + if n < 2 { + return n + } + return fibRef(n-1) + fibRef(n-2) +} + +func luaBudgetPick(rng *rand.Rand) luaapi.ExecutionBudgets { + var b luaapi.ExecutionBudgets + if rng.Intn(3) != 0 { + ticks := []int64{0, -1, 1, 2, 7, 64, 1000, math.MaxInt64} + b.TickBudget, b.TickBudgetSet = ticks[rng.Intn(len(ticks))], true + } + if rng.Intn(4) == 0 { + steps := []uint64{0, 3, 12, 40, 1_000_000} + b.MaxSteps, b.MaxStepsSet = steps[rng.Intn(len(steps))], true + } + return b +} + +// optionValue renders an integer option in one of the representations spawn +// options arrive in. +func optionValue(rng *rand.Rand, v int64) any { + switch rng.Intn(4) { + case 0: + return v + case 1: + return int(v) + case 2: + return float64(v) + default: + return json.Number(strconv.FormatInt(v, 10)) + } +} + +func luaChaosPlan(rng *rand.Rand, rec *luaRec) { + kind := rng.Intn(10) + incarnations := 1 + if kind <= 3 { + incarnations = 2 + rng.Intn(3) + } + for i := 0; i < incarnations; i++ { + var st luaStage + last := i == incarnations-1 + switch { + case last && kind == 4: + rec.failing = true + st.body = "local s = 0 for i = 1, 300 do s = s + i end\nerror('chaos-boom')\nlocal result = s" + case last && kind == 5: + rec.blocker = true + st.body = "local ch = channel.new(1)\nsubscribe('never', ch)\nch:receive()\nlocal result = 0" + case last && kind == 6: + rec.blocker = true + st.body = "local n = 0\nwhile true do n = n + 1 end\nlocal result = n" + default: + st.body, st.expected = luaStageBody(rng, rec) + } + st.entry = luaBudgetPick(rng) + if !last { + st.upgrade = true + if rng.Intn(3) == 0 { + // The replacement waits on a yield longer than the stale one. + st.staleMs = 8 + rng.Intn(8) + rec.wantsWake = true + } + } + rec.stages = append(rec.stages, st) + } + for i, st := range rec.stages { + if st.staleMs > 0 && i+1 < len(rec.stages) { + nxt := &rec.stages[i+1] + nxt.body = "do local r = test_yield(40) if tonumber(r) ~= 40 then error('wrong completion ' .. tostring(r) .. ' for 40') end end\n" + nxt.body + } + } + if rng.Intn(3) == 0 { + var spawn luaapi.ExecutionBudgets + bag := attrs.Bag{} + if rng.Intn(2) == 0 { + v := []int64{0, -1, 1, 5, 100, 5000}[rng.Intn(6)] + bag[luaapi.ProcessOptionTickBudget] = optionValue(rng, v) + spawn.TickBudget, spawn.TickBudgetSet = v, true + } + if rng.Intn(2) == 0 { + v := []int64{0, 4, 15, 60, 2_000_000}[rng.Intn(5)] + bag[luaapi.ProcessOptionMaxSteps] = optionValue(rng, v) + spawn.MaxSteps, spawn.MaxStepsSet = uint64(v), true + } + rec.spawn, rec.spawnBudget = bag, spawn + } +} + +type luaChaosLifecycle struct{ w *luaChaos } + +func (l luaChaosLifecycle) recFor(p pidapi.PID) *luaRec { + n, err := strconv.Atoi(strings.TrimPrefix(p.UniqID, "luachaos-")) + if err != nil || n < 0 || n >= len(l.w.recs) { + return nil + } + return l.w.recs[n] +} + +func (l luaChaosLifecycle) OnStart(_ context.Context, p pidapi.PID, _ process.Process) error { + if rec := l.recFor(p); rec != nil && rec.starts.Add(1) != 1 { + l.w.violatef("rec %d: started more than once", rec.id) + } + return nil +} + +func (l luaChaosLifecycle) OnComplete(_ context.Context, p pidapi.PID, res *runtime.Result) { + rec := l.recFor(p) + if rec == nil { + return + } + if rec.running.Load() != 0 { + l.w.violatef("rec %d: completed during a step", rec.id) + } + if rec.completes.Add(1) != 1 { + l.w.violatef("rec %d: completed more than once", rec.id) + return + } + rec.resultMu.Lock() + rec.result = res + rec.resultMu.Unlock() +} + +func (w *luaChaos) submit(rec *luaRec) { + proc, err := w.newProc(rec, 0) + if err != nil { + w.violatef("rec %d: cannot build the process: %v", rec.id, err) + rec.gaveUp.Store(true) + close(rec.ready) + return + } + frame, fc := ctxapi.OpenFrameContext(w.appCtx) + if err := runtime.SetFrameID(frame, registry.NewID("chaos", strconv.Itoa(rec.id))); err != nil { + w.violatef("rec %d: %v", rec.id, err) + } + if rec.spawn != nil { + if err := fc.Set(runtime.FrameLifecycleOptionsKey, attrs.Attributes(rec.spawn)); err != nil { + w.violatef("rec %d: %v", rec.id, err) + } + } + ctx, cancel := context.WithCancel(frame) + rec.cancel = cancel + _, err = w.sched.Submit(ctx, rec.pid, proc, "", nil) + if err != nil { + cancel() + proc.Close() + if !errors.Is(err, process.ErrSchedulerStopping) { + w.violatef("rec %d: unexpected Submit error %v", rec.id, err) + } + rec.gaveUp.Store(true) + close(rec.ready) + return + } + rec.submitted.Store(true) + close(rec.ready) +} + +func TestLuaChaos(t *testing.T) { + seed := int64(luaChaosEnvInt("WIPPY_LUA_CHAOS_SEED", 1, 1)) + runs := luaChaosEnvInt("WIPPY_LUA_CHAOS_RUNS", 3, 100) + for i := 0; i < runs; i++ { + s := seed + int64(i) + t.Run(fmt.Sprintf("seed=%d", s), func(t *testing.T) { runLuaChaos(t, s) }) + } +} + +func runLuaChaos(t *testing.T, seed int64) { + procs := luaChaosEnvInt("WIPPY_LUA_CHAOS_PROCS", 24, 150) + rng := rand.New(rand.NewSource(seed)) + workers := 1 + rng.Intn(4) + stopMode := rng.Intn(3) + t.Logf("lua chaos seed=%d workers=%d procs=%d stopMode=%d (replay: WIPPY_LUA_CHAOS_SEED=%d WIPPY_LUA_CHAOS_RUNS=1)", seed, workers, procs, stopMode, seed) + + baseline := goruntime.NumGoroutine() + w := &luaChaos{seed: seed} + reg := newMockRegistry() + reg.Register(testYieldCmdID, luaChaosHandler()) + w.sched = scheduler.NewScheduler(reg, scheduler.WithWorkers(workers), scheduler.WithLifecycle(luaChaosLifecycle{w: w})) + + appCtx := ctxapi.WithAppContext(context.Background(), ctxapi.NewAppContext()) + process.WithFactory(appCtx, luaChaosFactory{w: w}) + w.appCtx = appCtx + + for i := 0; i < procs; i++ { + rec := &luaRec{id: i, pid: pidapi.PID{Host: "test", UniqID: fmt.Sprintf("luachaos-%d", i)}, ready: make(chan struct{})} + rec.rng = rand.New(rand.NewSource(seed*1000003 + int64(i))) + luaChaosPlan(rec.rng, rec) + w.recs = append(w.recs, rec) + } + + w.sched.Start() + + var wg sync.WaitGroup + for _, rec := range w.recs { + wg.Add(1) + go func() { + defer wg.Done() + time.Sleep(time.Duration(rec.rng.Intn(30_000)) * time.Microsecond) + w.submit(rec) + }() + wg.Add(1) + go func() { + defer wg.Done() + <-rec.ready + if !rec.submitted.Load() { + return + } + r := rec.rng + deadline := time.Now().Add(20 * time.Second) + killAt := time.Now().Add(time.Duration(r.Intn(40)) * time.Millisecond) + kill := rec.blocker || r.Intn(8) == 0 + // Unconsumed messages stay queued in the actor, so the send rate backs off. + sent := 0 + for rec.completes.Load() == 0 && time.Now().Before(deadline) { + sent++ + if rec.wantsMsgs { + pkg := relay.NewPackage(pidapi.PID{}, rec.pid, "msg", payload.NewPayload(lua.LString("x"), payload.Lua)) + if err := w.sched.Send(pkg); err != nil && !errors.Is(err, process.ErrProcessNotFound) && !errors.Is(err, process.ErrProcessClosed) { + w.violatef("rec %d: unexpected Send error %v", rec.id, err) + } + } + if rec.wantsWake { + pkg := relay.NewPackage(pidapi.PID{}, rec.pid, "wake", payload.NewPayload(lua.LString("x"), payload.Lua)) + _ = w.sched.Send(pkg) + } + if kill && time.Now().After(killAt) { + rec.killed.Store(true) + if r.Intn(2) == 0 { + rec.cancel() + } else if err := w.sched.Terminate(rec.pid); err != nil && !errors.Is(err, process.ErrProcessNotFound) { + w.violatef("rec %d: unexpected Terminate error %v", rec.id, err) + } + return + } + time.Sleep(time.Duration(300+r.Intn(700)) * time.Microsecond * time.Duration(1+min(sent/50, 30))) + } + }() + } + + stopTimeout := 20 * time.Second + if stopMode == 2 { + stopTimeout = time.Duration(20+rng.Intn(80)) * time.Millisecond + } + stopDone := make(chan struct{}) + startStop := func() { + w.stopped.Store(true) + ctx, cancel := context.WithTimeout(context.Background(), stopTimeout) + w.sched.Stop(ctx) + cancel() + close(stopDone) + } + if stopMode != 0 { + time.AfterFunc(time.Duration(rng.Intn(80))*time.Millisecond, func() { go startStop() }) + } + + wg.Wait() + if stopMode == 0 { + w.awaitSettled(t, 60*time.Second) + go startStop() + } + select { + case <-stopDone: + case <-time.After(60 * time.Second): + t.Fatalf("seed %d: Stop did not return", seed) + } + w.awaitSettled(t, 10*time.Second) + w.verify(t, seed) + + deadline := time.Now().Add(10 * time.Second) + for goruntime.NumGoroutine() > baseline+2 && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + if n := goruntime.NumGoroutine(); n > baseline+2 { + buf := make([]byte, 1<<20) + t.Errorf("seed %d: %d goroutines remain, baseline %d\n%s", seed, n, baseline, buf[:goruntime.Stack(buf, true)]) + } +} + +func (w *luaChaos) awaitSettled(t *testing.T, d time.Duration) { + t.Helper() + deadline := time.Now().Add(d) + for { + var stranded []string + for _, rec := range w.recs { + select { + case <-rec.ready: + default: + stranded = append(stranded, fmt.Sprintf("rec %d never submitted", rec.id)) + continue + } + if rec.submitted.Load() && rec.completes.Load() == 0 { + stranded = append(stranded, fmt.Sprintf("rec %d (blocker=%v stages=%d) stranded", rec.id, rec.blocker, len(rec.stages))) + } + } + if len(stranded) == 0 { + return + } + if time.Now().After(deadline) { + t.Errorf("seed %d: %d actors stranded: %v\nprocesses: %+v", w.seed, len(stranded), stranded[:min(len(stranded), 10)], w.sched.ListProcesses()) + return + } + time.Sleep(5 * time.Millisecond) + } +} + +func (w *luaChaos) verify(t *testing.T, seed int64) { + t.Helper() + outcomes := map[string]int{} + for _, rec := range w.recs { + if rec.gaveUp.Load() { + outcomes["rejected"]++ + if rec.completes.Load() != 0 { + w.violatef("rec %d: completed although admission failed", rec.id) + } + continue + } + if n := rec.completes.Load(); n != 1 { + w.violatef("rec %d: completed %d times", rec.id, n) + continue + } + rec.resultMu.Lock() + res := rec.result + rec.resultMu.Unlock() + outcomes[w.checkOutcome(rec, res)]++ + } + t.Logf("seed %d outcomes %v", seed, outcomes) + + w.mu.Lock() + incs := append([]*chaosLua(nil), w.incs...) + w.mu.Unlock() + for _, p := range incs { + if p.inits.Load() > 0 && p.closed.Load() != 1 { + w.violatef("rec %d inc %d: closed %d times, want once", p.rec.id, p.inc, p.closed.Load()) + } + } + + stats := w.sched.Stats() + if stats["processes"] != 0 { + w.violatef("%d processes still registered after stop", stats["processes"]) + } + if stats["global_queue"] != 0 { + w.violatef("global queue holds %d entries after stop", stats["global_queue"]) + } + if len(w.violations) > 0 { + t.Errorf("seed %d: %d invariant violations (replay: WIPPY_LUA_CHAOS_SEED=%d WIPPY_LUA_CHAOS_RUNS=1):\n %s", + seed, len(w.violations), seed, strings.Join(w.violations, "\n ")) + } +} + +// checkOutcome verifies a finished actor and names its outcome class. +func (w *luaChaos) checkOutcome(rec *luaRec, res *runtime.Result) string { + if res == nil { + w.violatef("rec %d: no result", rec.id) + return "none" + } + killed := rec.killed.Load() || w.stopped.Load() + last := rec.stages[len(rec.stages)-1] + if res.Error == nil { + var got string + if res.Value != nil { + if v, ok := res.Value.Data().(lua.LValue); ok { + got = lua.LVAsString(v) + } + } + want := fmt.Sprintf("gen%d:%s", len(rec.stages)-1, last.expected) + if rec.failing || rec.blocker { + w.violatef("rec %d: finished with %q although its final stage cannot finish", rec.id, got) + } else if got != want { + w.violatef("rec %d: result %q, want %q", rec.id, got, want) + } + return "done" + } + switch { + case errors.Is(res.Error, process.ErrStepLimitExceeded): + if !rec.limited() { + w.violatef("rec %d: step limit error without max_steps", rec.id) + } + return "limit" + case rec.failing && strings.Contains(res.Error.Error(), "chaos-boom"): + return "fail" + case killed: + return "killed" + } + w.violatef("rec %d: unexpected error %v", rec.id, res.Error) + return "unexpected" +} diff --git a/runtime/lua/engine/factory_test.go b/runtime/lua/engine/factory_test.go index 53505cbf5..e99d14c57 100644 --- a/runtime/lua/engine/factory_test.go +++ b/runtime/lua/engine/factory_test.go @@ -7,6 +7,8 @@ import ( "testing" lua "github.com/wippyai/go-lua" + ctxapi "github.com/wippyai/runtime/api/context" + "github.com/wippyai/runtime/api/process" "github.com/wippyai/runtime/api/registry" luaapi "github.com/wippyai/runtime/api/runtime/lua" "github.com/wippyai/runtime/runtime/lua/code" @@ -781,8 +783,20 @@ func TestFactory_ModuleAliasing_LuaLibrary(t *testing.T) { t.Fatalf("factory() failed: %v", err) } - // Verify the alias is set correctly in the Lua state + // Library chunks run in the first step of the process. luaProc := proc.(*Process) + defer luaProc.Close() + frameCtx, _ := ctxapi.OpenFrameContext(context.Background()) + if err := luaProc.Init(frameCtx, "main", nil); err != nil { + t.Fatalf("Init failed: %v", err) + } + var output process.StepOutput + if err := luaProc.Step(nil, &output); err != nil { + t.Fatalf("Step failed: %v", err) + } + if v, _ := output.Result().Data().(lua.LValue); v == nil || v.String() != "from_utils" { + t.Fatalf("expected the entry point to read the library marker, got %v", output.Result().Data()) + } state := luaProc.State() // Imports are scoped to the chunk environment, not the shared _G. diff --git a/runtime/lua/engine/initializer.go b/runtime/lua/engine/initializer.go new file mode 100644 index 000000000..87f3514ec --- /dev/null +++ b/runtime/lua/engine/initializer.go @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: MPL-2.0 + +package engine + +import ( + lua "github.com/wippyai/go-lua" +) + +const initializersRegistryKey = "wippy.initializers" + +// Initializer is Lua code that must run before a process entry point, such as +// a library chunk whose result later chunks import. A ModuleBinder registers +// it with DeferInitializer instead of calling the function itself, so the code +// runs as a Lua frame of the process's main task: scheduled, preemptible and +// limited by the tick budget like the rest of the actor. +type Initializer struct { + // Fn is the function to run, created on the state the binder received. It + // runs without arguments. + Fn *lua.LFunction + // Done receives the first value Fn returned, or nil. It runs on the + // process's step goroutine, after Fn returns and before the next + // initializer starts. + Done func(result lua.LValue) +} + +// DeferInitializer registers init to run on l's process before its entry +// point. Initializers run in registration order, once per state: a later +// execution of the same process finds them completed. An initializer that +// fails is retried by the next execution. +// +// Call it from a ModuleBinder. Registration order must follow dependency +// order, so a library's imports are initialized before the library. +func DeferInitializer(l *lua.LState, init Initializer) { + list := ensureInitializers(l) + list.items = append(list.items, init) +} + +// initializerList is the registry-held sequence of a state's initializers. +// It implements lua.LValue so the state's registry can hold it. +type initializerList struct { + items []Initializer + // onComplete callbacks run once, when the last initializer completes. + onComplete []func() + // done counts completed initializers; started reports that advance has + // handed out items[done] in the current run. + done int + started bool +} + +func (*initializerList) String() string { return "" } +func (*initializerList) Type() lua.LValueType { return lua.LTUserData } + +// OnInitialized registers fn to run when l's deferred initializers have all +// completed, before the entry chunk runs. It does not run if none are +// registered with DeferInitializer. +func OnInitialized(l *lua.LState, fn func()) { + list := ensureInitializers(l) + list.onComplete = append(list.onComplete, fn) +} + +func initializersOf(l *lua.LState) *initializerList { + list, _ := l.G.Registry.RawGetString(initializersRegistryKey).(*initializerList) + return list +} + +func ensureInitializers(l *lua.LState) *initializerList { + list := initializersOf(l) + if list == nil { + list = &initializerList{} + l.G.Registry.RawSetString(initializersRegistryKey, list) + } + return list +} + +func (list *initializerList) pending() bool { + return list != nil && list.done < len(list.items) +} + +// begin restarts the run at the first incomplete initializer. +func (list *initializerList) begin() { + list.started = false +} + +// advance is the Go side of the bootstrap loop. It takes the previous +// initializer's result (ignored on the first call) and returns the next +// function to run, or nothing when all have run. +func (list *initializerList) advance(l *lua.LState) int { + if list.started { + list.complete(l.Get(1)) + } + list.started = list.done < len(list.items) + if !list.started { + return 0 + } + l.Push(list.items[list.done].Fn) + return 1 +} + +// runSync runs the incomplete initializers to completion on l's main thread, +// in order. It is the synchronous counterpart of the bootstrap loop for +// callers that execute without the scheduler; the code it runs is not +// preemptible. +func (list *initializerList) runSync(l *lua.LState) error { + if !list.pending() { + return nil + } + list.begin() + for list.done < len(list.items) { + item := list.items[list.done] + if err := l.CallByParam(lua.P{Fn: item.Fn, NRet: 1, Protect: true}); err != nil { + return err + } + result := l.Get(-1) + l.Pop(1) + list.complete(result) + } + return nil +} + +// complete records the result of the current initializer and drops it, so the +// state keeps neither the function nor its callback after the run. The last +// completion runs the one-shot callbacks and drops them as well. +func (list *initializerList) complete(result lua.LValue) { + item := list.items[list.done] + list.items[list.done] = Initializer{} + list.done++ + if item.Done != nil { + item.Done(result) + } + if list.done < len(list.items) { + return + } + callbacks := list.onComplete + list.onComplete = nil + for _, fn := range callbacks { + fn() + } +} diff --git a/runtime/lua/engine/initializer_release_test.go b/runtime/lua/engine/initializer_release_test.go new file mode 100644 index 000000000..b1e26c017 --- /dev/null +++ b/runtime/lua/engine/initializer_release_test.go @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: MPL-2.0 + +package engine + +import ( + "runtime" + "sync/atomic" + "testing" + "time" + + lua "github.com/wippyai/go-lua" +) + +// Completed initializers and one-shot completion callbacks are not retained by +// the state that outlives them. +func TestCompletedInitializersAreReleased(t *testing.T) { + var collected atomic.Int64 + probe := func() *finalizerProbe { + p := &finalizerProbe{collected: &collected} + runtime.SetFinalizer(p, func(p *finalizerProbe) { p.collected.Add(1) }) + return p + } + // The binder creates the captures itself: a binder that closed over them + // would keep them alive through the process's factory. + proc := mustNewProcess(t, + WithModuleBinder(wrapBinder(func(l *lua.LState) { + initProbe, doneProbe, callbackProbe := probe(), probe(), probe() + DeferInitializer(l, Initializer{ + Fn: l.NewFunction(func(*lua.LState) int { _ = initProbe; return 0 }), + Done: func(lua.LValue) { _ = doneProbe }, + }) + OnInitialized(l, func() { _ = callbackProbe }) + })), + WithScript(`return { main = function() return 1 end }`, "m.lua")) + t.Cleanup(proc.Close) + + runProcess(t, proc, 1) + + deadline := time.Now().Add(5 * time.Second) + for collected.Load() < 3 && time.Now().Before(deadline) { + runtime.GC() + time.Sleep(10 * time.Millisecond) + } + if got := collected.Load(); got != 3 { + t.Fatalf("%d of 3 completed initializer captures were collected", got) + } +} diff --git a/runtime/lua/engine/initializer_test.go b/runtime/lua/engine/initializer_test.go new file mode 100644 index 000000000..2503dacd6 --- /dev/null +++ b/runtime/lua/engine/initializer_test.go @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: MPL-2.0 + +package engine + +import ( + "testing" + + lua "github.com/wippyai/go-lua" + "github.com/wippyai/runtime/api/process" +) + +// An initializer that fails is retried by the next execution of the process; +// completion callbacks run once, after the last initializer succeeds. +func TestFailedInitializerIsRetriedByNextExecution(t *testing.T) { + var attempts, done, initialized int + proc := mustNewProcess(t, + WithModuleBinder(wrapBinder(func(l *lua.LState) { + DeferInitializer(l, Initializer{ + Fn: l.NewFunction(func(l *lua.LState) int { + attempts++ + if attempts == 1 { + l.RaiseError("init failed") + } + l.Push(lua.LNumber(attempts)) + return 1 + }), + Done: func(lua.LValue) { done++ }, + }) + OnInitialized(l, func() { initialized++ }) + })), + WithScript(`return { main = function() return 1 end }`, "m.lua")) + t.Cleanup(proc.Close) + + if err := proc.Init(frameContext(), "main", nil); err != nil { + t.Fatal(err) + } + var output process.StepOutput + if err := proc.Step(nil, &output); err == nil { + t.Fatal("expected the failing initializer to fail the step") + } + if done != 0 || initialized != 0 { + t.Fatalf("failed initializer completed: done=%d initialized=%d", done, initialized) + } + + runProcess(t, proc, 1) + if attempts != 2 || done != 1 || initialized != 1 { + t.Fatalf("attempts=%d done=%d initialized=%d", attempts, done, initialized) + } +} + +// A missing method fails with a not-found error after the initializers +// completed, and a later execution does not run them again. +func TestMethodNotFoundAfterInitializers(t *testing.T) { + var runs, done int + proc := mustNewProcess(t, + WithModuleBinder(wrapBinder(func(l *lua.LState) { + DeferInitializer(l, Initializer{ + Fn: l.NewFunction(func(*lua.LState) int { runs++; return 0 }), + Done: func(lua.LValue) { done++ }, + }) + })), + WithScript(`return { main = function() return 1 end }`, "m.lua")) + t.Cleanup(proc.Close) + + if err := proc.Init(frameContext(), "missing", nil); err != nil { + t.Fatal(err) + } + var output process.StepOutput + requireMethodNotFound(t, proc.Step(nil, &output)) + if runs != 1 || done != 1 { + t.Fatalf("initializer runs=%d done=%d, want 1 and 1", runs, done) + } + + runProcess(t, proc, 1) + if runs != 1 { + t.Fatalf("completed initializer ran again: %d", runs) + } +} diff --git a/runtime/lua/engine/preempt_leak_test.go b/runtime/lua/engine/preempt_leak_test.go new file mode 100644 index 000000000..a62d3132d --- /dev/null +++ b/runtime/lua/engine/preempt_leak_test.go @@ -0,0 +1,306 @@ +// SPDX-License-Identifier: MPL-2.0 + +package engine + +import ( + "context" + "errors" + "fmt" + "reflect" + "runtime" + "sync" + "sync/atomic" + "testing" + "time" + + lua "github.com/wippyai/go-lua" + "github.com/wippyai/runtime/api/attrs" + ctxapi "github.com/wippyai/runtime/api/context" + "github.com/wippyai/runtime/api/process" + "github.com/wippyai/runtime/api/registry" + "github.com/wippyai/runtime/api/relay" + apiruntime "github.com/wippyai/runtime/api/runtime" + luaapi "github.com/wippyai/runtime/api/runtime/lua" + "github.com/wippyai/runtime/runtime/lua/code" + scheduler "github.com/wippyai/runtime/system/scheduler/actor" +) + +func heapInUse() uint64 { + runtime.GC() + runtime.GC() + var m runtime.MemStats + runtime.ReadMemStats(&m) + return m.HeapAlloc +} + +func settleGoroutines(want int) int { + n := 0 + for i := 0; i < 200; i++ { + runtime.GC() + if n = runtime.NumGoroutine(); n <= want { + return n + } + time.Sleep(10 * time.Millisecond) + } + return n +} + +func libraryCompiled(t *testing.T) *code.CompiledMain { + t.Helper() + baseID := registry.NewID("test", "base") + midID := registry.NewID("test", "mid") + mainID := registry.NewID("test", "main") + return &code.CompiledMain{ + MainID: mainID, + Imports: map[registry.ID][]code.Import{ + midID: {{ID: baseID, Alias: "b"}}, + mainID: {{ID: midID, Alias: "m"}}, + }, + Dependencies: []code.CompiledProto{ + {Name: "base", Node: &code.Node{ID: baseID}, Proto: compileTestProto(t, `return { v = 1 }`)}, + {Name: "mid", Node: &code.Node{ID: midID}, Proto: compileTestProto(t, `return { v = b.v + 1 }`)}, + }, + } +} + +// Repeated executions of one process neither add initializers, exported +// functions or registry entries nor grow the heap. +func TestPooledExecutionsDoNotGrowState(t *testing.T) { + proc := newDependencyProcess(t, libraryCompiled(t), `return { main = function() return m.v end }`, 50) + proc.EnablePreemption() + + run := func(n int) { + for i := 0; i < n; i++ { + if err := proc.Init(frameContext(), "main", nil); err != nil { + t.Fatal(err) + } + var output process.StepOutput + for j := 0; ; j++ { + output.Reset() + if err := proc.Step(nil, &output); err != nil { + t.Fatal(err) + } + if output.Status() == process.StepDone { + break + } + if j > 100 { + t.Fatal("execution did not finish") + } + } + } + } + + run(50) + list := initializersOf(proc.State()) + items, complete := len(list.items), len(list.onComplete) + exported := len(proc.exported) + top := proc.State().GetTop() + heap0 := heapInUse() + + for batch := 0; batch < 4; batch++ { + run(2000) + } + + list = initializersOf(proc.State()) + if len(list.items) != items || len(list.onComplete) != complete { + t.Fatalf("initializer list grew: items %d->%d, onComplete %d->%d", items, len(list.items), complete, len(list.onComplete)) + } + if len(proc.exported) != exported { + t.Fatalf("exported cache grew: %d->%d", exported, len(proc.exported)) + } + if got := proc.State().GetTop(); got != top { + t.Fatalf("main stack grew: %d->%d", top, got) + } + if len(proc.threads) != 0 { + t.Fatalf("%d threads retained", len(proc.threads)) + } + heap1 := heapInUse() + if heap1 > heap0+heap0/2+1<<20 { + t.Fatalf("heap grew from %d to %d over 8000 executions", heap0, heap1) + } +} + +type finalizerProbe struct{ collected *atomic.Int64 } + +// Process.Close drops every reference this package adds, so a closed process +// keeps nothing of its Lua state alive and its fields are reset for reuse. +func TestCloseReleasesPreemptedProcessState(t *testing.T) { + var collected atomic.Int64 + const total = 200 + baseline := runtime.NumGoroutine() + + for i := 0; i < total; i++ { + probe := &finalizerProbe{collected: &collected} + runtime.SetFinalizer(probe, func(p *finalizerProbe) { p.collected.Add(1) }) + binder := wrapBinder(func(l *lua.LState) { + ud := l.NewUserData() + ud.Value = probe + l.SetGlobal("probe", ud) + }) + entry := luaapi.ExecutionBudgets{TickBudget: 20, TickBudgetSet: true, MaxSteps: 1000, MaxStepsSet: true} + proc := mustNewProcess(t, + WithModuleBinder(binder), + WithProcessExecutionBudgets(entry), + WithScript(` + local t = {} + local co = coroutine.wrap(function() while true do t[#t+1] = probe coroutine.yield() end end) + return { main = function() for i = 1, 1e9 do co() end end } + `, "spin.lua")) + proc.EnablePreemption() + if err := proc.Init(frameContext(), "main", nil); err != nil { + t.Fatal(err) + } + var output process.StepOutput + for j := 0; j < 5; j++ { + output.Reset() + if err := proc.Step(nil, &output); err != nil { + t.Fatal(err) + } + if output.Status() != process.StepPreempted { + t.Fatalf("expected preemption, got %v", output.Status()) + } + } + proc.Close() + + if proc.preempted != nil || proc.exportFn != nil || proc.exported != nil || proc.steps != 0 || + proc.budgets != (luaapi.ExecutionBudgets{}) || proc.entryBudgets != (luaapi.ExecutionBudgets{}) || proc.preemptive { + t.Fatalf("closed process retains state: %+v", proc) + } + v := reflect.ValueOf(proc).Elem() + for _, name := range []string{"mainTask", "state", "factory", "upgradeRequest", "pendingOutdated", "result", "execErr"} { + if f := v.FieldByName(name); f.IsValid() && !f.IsNil() { + t.Fatalf("closed process retains %s", name) + } + } + } + + deadline := time.Now().Add(10 * time.Second) + for collected.Load() < total && time.Now().Before(deadline) { + runtime.GC() + time.Sleep(10 * time.Millisecond) + } + if got := collected.Load(); got != total { + t.Fatalf("%d of %d closed processes were collected", got, total) + } + if got := settleGoroutines(baseline); got > baseline { + t.Fatalf("goroutines grew from %d to %d", baseline, got) + } +} + +// Closing a process with drained initializers, a preempted task and queued +// messages across many cycles keeps the heap flat. +func TestSpawnPreemptCloseCyclesKeepHeapFlat(t *testing.T) { + compiled := libraryCompiled(t) + cycle := func(n int) { + for i := 0; i < n; i++ { + binder := NewProcessFactory(nil).isolationBinder(compiled, newProcessConfig(), nil, nil, nil, nil) + proc := mustNewProcess(t, + WithModuleBinder(wrapBinder(func(l *lua.LState) { lua.OpenErrors(l) })), + WithModuleBinder(binder), + WithProcessExecutionBudgets(luaapi.ExecutionBudgets{TickBudget: 10, TickBudgetSet: true}), + WithScript(fmt.Sprintf(`return { main = function() local n = %d while true do n = n + m.v end end }`, i), "m.lua")) + proc.EnablePreemption() + if err := proc.Init(frameContext(), "main", nil); err != nil { + t.Fatal(err) + } + var output process.StepOutput + for j := 0; j < 6; j++ { + output.Reset() + if err := proc.Step(nil, &output); err != nil { + t.Fatal(err) + } + } + proc.Close() + } + } + cycle(200) + heap0 := heapInUse() + for batch := 0; batch < 4; batch++ { + cycle(500) + } + heap1 := heapInUse() + if heap1 > heap0+heap0/2+1<<20 { + t.Fatalf("heap grew from %d to %d over 2000 cycles", heap0, heap1) + } +} + +type closeCounting struct { + *Process + closed *atomic.Int64 +} + +func (c *closeCounting) Close() { + c.closed.Add(1) + c.Process.Close() +} + +func (c *closeCounting) Send(*relay.Package) error { return nil } + +// Actors that finish, exceed max_steps or are terminated while preempting are +// all closed, unregistered and leave no goroutines behind. +func TestSchedulerLuaActorExitsReleaseEverything(t *testing.T) { + baseline := runtime.NumGoroutine() + var closed atomic.Int64 + executor := newBenchExecutorWithOptions(newMockRegistry(), scheduler.WithWorkers(4)) + executor.Start() + + finish := compileTestProto(t, `local s = 0 for i = 1, 3000 do s = s + i end return s`) + spin := compileTestProto(t, `while true do end`) + budgets := luaapi.ExecutionBudgets{TickBudget: 50, TickBudgetSet: true} + + const total = 300 + var wg sync.WaitGroup + results := make([]error, total) + for i := 0; i < total; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + proto := finish + if i%3 != 0 { + proto = spin + } + proc := mustNewProcess(t, WithProto(proto), WithProcessExecutionBudgets(budgets)) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + frame, fc := ctxapi.OpenFrameContext(ctx) + switch i % 3 { + case 1: + _ = fc.Set(apiruntime.FrameLifecycleOptionsKey, attrs.Attributes(attrs.Bag{"max_steps": int64(10)})) + case 2: + time.AfterFunc(20*time.Millisecond, cancel) + } + result, err := executor.Execute(frame, newTestPID(fmt.Sprintf("a%d", i)), &closeCounting{Process: proc, closed: &closed}, "", nil) + if err != nil { + results[i] = err + return + } + results[i] = result.Error + }(i) + } + wg.Wait() + + for i, err := range results { + switch i % 3 { + case 0: + if err != nil { + t.Fatalf("actor %d: %v", i, err) + } + case 1: + if !errors.Is(err, process.ErrStepLimitExceeded) { + t.Fatalf("actor %d: expected step limit, got %v", i, err) + } + case 2: + if err == nil { + t.Fatalf("actor %d: expected termination", i) + } + } + } + // Stopping joins the workers, so every Close has happened. + executor.Stop() + if got := closed.Load(); got != total { + t.Fatalf("closed %d of %d actors", got, total) + } + if got := settleGoroutines(baseline); got > baseline { + t.Fatalf("goroutines grew from %d to %d", baseline, got) + } +} diff --git a/runtime/lua/engine/preempt_test.go b/runtime/lua/engine/preempt_test.go new file mode 100644 index 000000000..3726754c4 --- /dev/null +++ b/runtime/lua/engine/preempt_test.go @@ -0,0 +1,718 @@ +// SPDX-License-Identifier: MPL-2.0 + +package engine + +import ( + "context" + "errors" + "fmt" + "math" + "sync/atomic" + "testing" + "time" + + lua "github.com/wippyai/go-lua" + "github.com/wippyai/runtime/api/attrs" + ctxapi "github.com/wippyai/runtime/api/context" + "github.com/wippyai/runtime/api/process" + "github.com/wippyai/runtime/api/registry" + "github.com/wippyai/runtime/api/relay" + "github.com/wippyai/runtime/api/runtime" + luaapi "github.com/wippyai/runtime/api/runtime/lua" + scheduler "github.com/wippyai/runtime/system/scheduler/actor" +) + +var _ process.Preemptible = (*Process)(nil) + +// stepPreempted runs proc until it completes and returns its result value and +// the number of steps that reported StepPreempted. +func stepPreempted(t *testing.T, proc *Process, maxSteps int) (lua.LValue, int) { + t.Helper() + var output process.StepOutput + preempts := 0 + for i := 0; i < maxSteps; i++ { + output.Reset() + if err := proc.Step(nil, &output); err != nil { + t.Fatalf("step %d: %v", i, err) + } + switch output.Status() { + case process.StepPreempted: + preempts++ + case process.StepDone: + if output.Result() == nil { + t.Fatal("expected a result") + } + v, _ := output.Result().Data().(lua.LValue) + return v, preempts + case process.StepContinue: + default: + t.Fatalf("step %d: unexpected status %v", i, output.Status()) + } + } + t.Fatalf("not done after %d steps (%d preemptions)", maxSteps, preempts) + return nil, 0 +} + +// initPreemptProcess starts script with the given entry tick_budget under a +// scheduler that enables preemption; budget 0 leaves tick_budget unset. +func initPreemptProcess(t *testing.T, script string, budget int64) *Process { + t.Helper() + var budgets luaapi.ExecutionBudgets + if budget != 0 { + budgets = luaapi.ExecutionBudgets{TickBudget: budget, TickBudgetSet: true} + } + proc := mustNewProcess(t, WithScript(script, "preempt.lua"), WithProcessExecutionBudgets(budgets)) + proc.EnablePreemption() + ctx, _ := ctxapi.OpenFrameContext(context.Background()) + if err := proc.Init(ctx, "", nil); err != nil { + t.Fatalf("init: %v", err) + } + t.Cleanup(proc.Close) + return proc +} + +func TestProcessPreemptsLongRunningStep(t *testing.T) { + proc := initPreemptProcess(t, ` + local s = 0 + for i = 1, 100000 do s = s + i end + return s + `, 1000) + + v, preempts := stepPreempted(t, proc, 10000) + if lua.LVAsNumber(v) != 5000050000 { + t.Fatalf("expected 5000050000, got %v", v) + } + if preempts < 50 { + t.Fatalf("expected the loop to be split into slices, got %d preemptions", preempts) + } +} + +func TestCPUPreemptionDoesNotAllocatePendingYields(t *testing.T) { + proc := initPreemptProcess(t, `while true do end`, 100) + var output process.StepOutput + for i := 0; i < 10; i++ { + output.Reset() + if err := proc.Step(nil, &output); err != nil { + t.Fatal(err) + } + if output.Status() != process.StepPreempted { + t.Fatalf("expected preemption, got %v", output.Status()) + } + if proc.pendingYields != nil { + t.Fatal("CPU-only preemption allocated external-yield bookkeeping") + } + } +} + +func TestProcessNegativeTickBudgetRunsStepToCompletion(t *testing.T) { + proc := initPreemptProcess(t, ` + local s = 0 + for i = 1, 100000 do s = s + i end + return s + `, -1) + + var output process.StepOutput + if err := proc.Step(nil, &output); err != nil { + t.Fatal(err) + } + if output.Status() != process.StepDone { + t.Fatalf("expected StepDone in one step, got %v", output.Status()) + } +} + +func TestProcessTickBudgetBoundaries(t *testing.T) { + const script = ` + local s = 0 + for i = 1, 100000 do s = s + i end + return s + ` + t.Run("max int64 runs the step to completion", func(t *testing.T) { + proc := initPreemptProcess(t, script, math.MaxInt64) + var output process.StepOutput + if err := proc.Step(nil, &output); err != nil { + t.Fatal(err) + } + if output.Status() != process.StepDone { + t.Fatalf("expected StepDone in one step, got %v", output.Status()) + } + }) + t.Run("budget of one makes progress", func(t *testing.T) { + proc := initPreemptProcess(t, `return 7`, 1) + if v, _ := stepPreempted(t, proc, 100); lua.LVAsNumber(v) != 7 { + t.Fatalf("expected 7, got %v", v) + } + }) +} + +// A budget that runs out exactly as the last task ends neither loses the +// result nor reports a preemption with nothing left to run. +func TestProcessBudgetExhaustedAtTaskEnd(t *testing.T) { + const script = ` + local function f(n) return n + 1 end + local s = 0 + for i = 1, 5 do s = f(s) end + return s + ` + for budget := int64(1); budget <= 40; budget++ { + proc := initPreemptProcess(t, script, budget) + v, _ := stepPreempted(t, proc, 1000) + if lua.LVAsNumber(v) != 5 { + t.Fatalf("budget %d: expected 5, got %v", budget, v) + } + } +} + +// The budget covers the whole step: coroutines that did not get to run +// before it ran out are deferred to the next step rather than each getting a +// fresh budget. +func TestProcessPreemptionBudgetIsPerStep(t *testing.T) { + proc := initPreemptProcess(t, ` + local done = 0 + local function work() + local s = 0 + for i = 1, 20000 do s = s + i end + done = done + 1 + end + for _ = 1, 8 do coroutine.spawn(work) end + while done < 8 do coroutine.yield() end + return done + `, 500) + + var output process.StepOutput + steps := 0 + for { + steps++ + if steps > 100000 { + t.Fatal("process did not complete") + } + output.Reset() + if err := proc.Step(nil, &output); err != nil { + t.Fatal(err) + } + if output.Status() == process.StepDone { + break + } + } + v, _ := output.Result().Data().(lua.LValue) + if lua.LVAsNumber(v) != 8 { + t.Fatalf("expected 8 finished coroutines, got %v", v) + } + // 8 coroutines * 20000 iterations need at least 320 slices of 500 ticks. + if steps < 320 { + t.Fatalf("expected per-step slicing across coroutines, completed in %d steps", steps) + } +} + +func TestProcessPreemptionKeepsSpawnedCoroutinesConsistent(t *testing.T) { + proc := initPreemptProcess(t, ` + local total = 0 + local function add(n) + local s = 0 + for i = 1, n do s = s + i end + total = total + s + end + coroutine.spawn(function() add(5000) end) + coroutine.spawn(function() add(3000) end) + add(1000) + while total ~= 12502500 + 4501500 + 500500 do coroutine.yield() end + return total + `, 300) + + v, preempts := stepPreempted(t, proc, 100000) + if lua.LVAsNumber(v) != 17504500 { + t.Fatalf("expected 17504500, got %v", v) + } + if preempts == 0 { + t.Fatal("expected preemption") + } +} + +// On a single scheduler worker, a Lua actor stuck in an endless loop is +// preempted so another actor still runs to completion. +func TestSpinningActorDoesNotStarveOtherActors(t *testing.T) { + executor := newBenchExecutorWithOptions(newMockRegistry(), + scheduler.WithWorkers(1)) + executor.Start() + defer executor.Stop() + + spinProto, err := lua.CompileString(`while true do end`, "spin.lua") + if err != nil { + t.Fatal(err) + } + quickProto, err := lua.CompileString(`local s = 0 for i = 1, 1000 do s = s + i end return s`, "quick.lua") + if err != nil { + t.Fatal(err) + } + + spinCtx, stopSpin := context.WithCancel(context.Background()) + defer stopSpin() + spinFrame, _ := ctxapi.OpenFrameContext(spinCtx) + spin := &enteredProcess{Process: mustNewProcess(t, WithProto(spinProto)), entered: make(chan struct{})} + spinDone := make(chan struct{}) + go func() { + defer close(spinDone) + _, _ = executor.Execute(spinFrame, newTestPID("spin"), spin, "", nil) + }() + select { + case <-spin.entered: + case <-time.After(5 * time.Second): + t.Fatal("spinning actor did not start") + } + + quickCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + quickFrame, _ := ctxapi.OpenFrameContext(quickCtx) + result, err := executor.Execute(quickFrame, newTestPID("quick"), &mockProcess{Process: mustNewProcess(t, WithProto(quickProto))}, "", nil) + if err != nil { + t.Fatalf("quick actor did not complete while another actor spins: %v", err) + } + if result.Error != nil { + t.Fatalf("quick actor failed: %v", result.Error) + } + if v, _ := result.Value.Data().(lua.LValue); lua.LVAsNumber(v) != 500500 { + t.Fatalf("expected 500500, got %v", result.Value.Data()) + } + + select { + case <-spinDone: + t.Fatal("spinning actor finished unexpectedly") + default: + } + stopSpin() + select { + case <-spinDone: + case <-time.After(5 * time.Second): + t.Fatal("spinning actor did not stop after cancellation") + } +} + +// enteredProcess signals when its first step begins. +type enteredProcess struct { + *Process + entered chan struct{} + started bool +} + +func (p *enteredProcess) Step(events []process.Event, out *process.StepOutput) error { + if !p.started { + p.started = true + close(p.entered) + } + return p.Process.Step(events, out) +} + +// Without a scheduler that enables preemption a step runs to completion +// whatever the tick budget. +func TestProcessNotEnabledForPreemptionRunsStepToCompletion(t *testing.T) { + proc := mustNewProcess(t, WithScript(` + local s = 0 + for i = 1, 100000 do s = s + i end + return s + `, "preempt.lua"), WithProcessExecutionBudgets(luaapi.ExecutionBudgets{TickBudget: 10, TickBudgetSet: true})) + ctx, _ := ctxapi.OpenFrameContext(context.Background()) + if err := proc.Init(ctx, "", nil); err != nil { + t.Fatal(err) + } + defer proc.Close() + var output process.StepOutput + if err := proc.Step(nil, &output); err != nil { + t.Fatal(err) + } + if output.Status() != process.StepDone { + t.Fatalf("expected StepDone in one step, got %v", output.Status()) + } +} + +func TestProcessDefaultTickBudget(t *testing.T) { + proc := initPreemptProcess(t, ` + local s = 0 + for i = 1, 100000 do s = s + i end + return s + `, 0) + _, preempts := stepPreempted(t, proc, 100000) + // 100000 iterations at the default budget of 512 ticks. + if want := 100000 / int(DefaultActorTickBudget); preempts < want { + t.Fatalf("expected at least %d preemptions at the default budget, got %d", want, preempts) + } +} + +// initWithSpawnOptions starts script with spawn options in the frame context, +// as the host does for process.with_options(...) spawns. +func initWithSpawnOptions(t *testing.T, script string, entry luaapi.ExecutionBudgets, spawn attrs.Bag) (*Process, error) { + t.Helper() + proc := mustNewProcess(t, WithScript(script, "preempt.lua"), WithProcessExecutionBudgets(entry)) + proc.EnablePreemption() + t.Cleanup(proc.Close) + ctx, fc := ctxapi.OpenFrameContext(context.Background()) + if err := fc.Set(runtime.FrameLifecycleOptionsKey, attrs.Attributes(spawn)); err != nil { + t.Fatal(err) + } + return proc, proc.Init(ctx, "", nil) +} + +func TestProcessSpawnOptionsOverrideEntryTickBudget(t *testing.T) { + script := ` + local s = 0 + for i = 1, 100000 do s = s + i end + return s + ` + entry := luaapi.ExecutionBudgets{TickBudget: 100, TickBudgetSet: true} + proc, err := initWithSpawnOptions(t, script, entry, attrs.Bag{"tick_budget": int64(-1)}) + if err != nil { + t.Fatal(err) + } + var output process.StepOutput + if err := proc.Step(nil, &output); err != nil { + t.Fatal(err) + } + if output.Status() != process.StepDone { + t.Fatalf("spawn tick_budget=-1 must disable preemption, got %v", output.Status()) + } +} + +func TestProcessRejectsInvalidSpawnOptions(t *testing.T) { + cases := []attrs.Bag{ + {"tick_budget": "fast"}, + {"tick_budget": 1.5}, + {"max_steps": int64(-1)}, + } + for _, spawn := range cases { + if _, err := initWithSpawnOptions(t, `return 1`, luaapi.ExecutionBudgets{}, spawn); err == nil { + t.Fatalf("expected %v to be rejected", spawn) + } + } +} + +func TestProcessMaxStepsFailsProcess(t *testing.T) { + script := ` + local s = 0 + for i = 1, 100000 do s = s + i end + return s + ` + entry := luaapi.ExecutionBudgets{TickBudget: 100, TickBudgetSet: true, MaxSteps: 5, MaxStepsSet: true} + proc, err := initWithSpawnOptions(t, script, entry, nil) + if err != nil { + t.Fatal(err) + } + var output process.StepOutput + for i := 0; i < 5; i++ { + output.Reset() + if err := proc.Step(nil, &output); err != nil { + t.Fatalf("step %d: %v", i, err) + } + if output.Status() != process.StepPreempted { + t.Fatalf("step %d: expected StepPreempted, got %v", i, output.Status()) + } + } + output.Reset() + if err := proc.Step(nil, &output); !errors.Is(err, process.ErrStepLimitExceeded) { + t.Fatalf("expected ErrStepLimitExceeded, got %v", err) + } +} + +type countingLease struct{ released atomic.Int32 } + +func (l *countingLease) Release() { l.released.Add(1) } + +// A step that exceeds max_steps fails the process, and the retention leases of +// the messages delivered with that step are released. +func TestProcessMaxStepsReleasesDeliveredMessages(t *testing.T) { + script := ` + local s = 0 + for i = 1, 100000 do s = s + i end + return s + ` + entry := luaapi.ExecutionBudgets{TickBudget: 100, TickBudgetSet: true, MaxSteps: 1, MaxStepsSet: true} + // The test closes the process itself to observe the lease release, so it + // does not use initWithSpawnOptions, which closes it at cleanup. + proc := mustNewProcess(t, WithScript(script, "preempt.lua"), WithProcessExecutionBudgets(entry)) + proc.EnablePreemption() + ctx, _ := ctxapi.OpenFrameContext(context.Background()) + if err := proc.Init(ctx, "", nil); err != nil { + t.Fatal(err) + } + var output process.StepOutput + if err := proc.Step(nil, &output); err != nil { + t.Fatal(err) + } + + lease := &countingLease{} + msg := relay.AcquireMessage() + msg.Topic = "inbox" + msg.SetRetentionLease(lease) + pkg := relay.AcquirePackage() + pkg.Messages = append(pkg.Messages, msg) + + output.Reset() + err := proc.Step([]process.Event{{Type: process.EventMessage, Data: pkg}}, &output) + if !errors.Is(err, process.ErrStepLimitExceeded) { + t.Fatalf("expected ErrStepLimitExceeded, got %v", err) + } + proc.Close() + if got := lease.released.Load(); got != 1 { + t.Fatalf("expected the retention lease to be released once, got %d", got) + } +} + +// Tasks that did not get a turn in an exhausted step stay queued in order; the +// step does not walk the run queue. +func TestProcessExhaustedBudgetKeepsWaitingTasksQueued(t *testing.T) { + proc := initPreemptProcess(t, ` + local release = false + for _ = 1, 50 do + coroutine.spawn(function() while not release do end end) + end + while true do coroutine.yield() end + `, 20) + + var output process.StepOutput + if err := proc.Step(nil, &output); err != nil { + t.Fatal(err) + } + if output.Status() != process.StepPreempted { + t.Fatalf("expected StepPreempted, got %v", output.Status()) + } + if proc.preempted != nil { + t.Fatal("the preempted task must be back in the run queue at the end of the step") + } + if got := proc.queue.Len(); got != len(proc.threads) { + t.Fatalf("expected every live task queued (%d threads), got %d queued", len(proc.threads), got) + } +} + +// Module top-level code runs inside the first step under the tick budget, so +// Init returns at once and an endless chunk is preempted like any other loop. +func TestProcessModuleTopLevelIsPreempted(t *testing.T) { + proc := mustNewProcess(t, WithScript(` + while true do end + return { main = function() end } + `, "module.lua"), WithProcessExecutionBudgets(luaapi.ExecutionBudgets{TickBudget: 100, TickBudgetSet: true})) + proc.EnablePreemption() + t.Cleanup(proc.Close) + ctx, _ := ctxapi.OpenFrameContext(context.Background()) + + inited := make(chan error, 1) + go func() { inited <- proc.Init(ctx, "main", nil) }() + select { + case err := <-inited: + if err != nil { + t.Fatal(err) + } + case <-time.After(5 * time.Second): + t.Fatal("Init blocked on module top-level code") + } + + var output process.StepOutput + if err := proc.Step(nil, &output); err != nil { + t.Fatal(err) + } + if output.Status() != process.StepPreempted { + t.Fatalf("expected the module chunk to be preempted, got %v", output.Status()) + } +} + +// A module whose top level never ends does not block the goroutine that +// submits it to the scheduler. +func TestSubmitDoesNotRunModuleTopLevelOnCaller(t *testing.T) { + executor := newBenchExecutorWithOptions(newMockRegistry(), scheduler.WithWorkers(1)) + executor.Start() + defer executor.Stop() + + proc := mustNewProcess(t, WithScript(` + while true do end + return { main = function() end } + `, "module.lua")) + ctx, cancelProc := context.WithCancel(context.Background()) + defer cancelProc() + frame, _ := ctxapi.OpenFrameContext(ctx) + + submitted := make(chan error, 1) + go func() { + _, err := executor.sched.Submit(frame, newTestPID("module-spin"), &mockProcess{Process: proc}, "main", nil) + submitted <- err + }() + select { + case err := <-submitted: + if err != nil { + t.Fatal(err) + } + case <-time.After(5 * time.Second): + t.Fatal("Submit blocked on module top-level code") + } +} + +// The module chunk runs once per process; later executions of the same +// process reuse the cached method. +func TestProcessReusesCachedModuleAcrossExecutions(t *testing.T) { + proc := mustNewProcess(t, WithScript(` + runs = (runs or 0) + 1 + return { main = function() return runs end } + `, "module.lua")) + t.Cleanup(proc.Close) + + for i := 0; i < 3; i++ { + ctx, _ := ctxapi.OpenFrameContext(context.Background()) + if err := proc.Init(ctx, "main", nil); err != nil { + t.Fatalf("execution %d: %v", i, err) + } + var output process.StepOutput + if err := proc.Step(nil, &output); err != nil { + t.Fatalf("execution %d: %v", i, err) + } + if output.Status() != process.StepDone { + t.Fatalf("execution %d: status %v", i, output.Status()) + } + if v, _ := output.Result().Data().(lua.LValue); lua.LVAsNumber(v) != 1 { + t.Fatalf("execution %d: expected the module chunk to have run once, got %v", i, output.Result().Data()) + } + } +} + +func TestProcessStepCountContinuesAcrossResume(t *testing.T) { + script := ` + local s = 0 + for i = 1, 100000 do s = s + i end + return s + ` + entry := luaapi.ExecutionBudgets{TickBudget: 100, TickBudgetSet: true, MaxSteps: 5, MaxStepsSet: true} + proc, err := initWithSpawnOptions(t, script, entry, nil) + if err != nil { + t.Fatal(err) + } + proc.ResumeStepCount(4) + if proc.StepsUsed() != 4 { + t.Fatalf("expected 4 steps used, got %d", proc.StepsUsed()) + } + var output process.StepOutput + if err := proc.Step(nil, &output); err != nil { + t.Fatalf("the fifth step is within the limit: %v", err) + } + output.Reset() + if err := proc.Step(nil, &output); !errors.Is(err, process.ErrStepLimitExceeded) { + t.Fatalf("expected ErrStepLimitExceeded, got %v", err) + } +} + +func TestProcessStepsUsedCountsStepsWithoutLimit(t *testing.T) { + script := ` + local s = 0 + for i = 1, 100000 do s = s + i end + return s + ` + entry := luaapi.ExecutionBudgets{TickBudget: 100, TickBudgetSet: true} + proc, err := initWithSpawnOptions(t, script, entry, nil) + if err != nil { + t.Fatal(err) + } + var output process.StepOutput + for i := 0; i < 3; i++ { + output.Reset() + if err := proc.Step(nil, &output); err != nil { + t.Fatal(err) + } + } + if got := proc.StepsUsed(); got != 3 { + t.Fatalf("expected 3 steps used, got %d", got) + } +} + +type upgradeFactory struct { + create func() (process.Process, *process.Meta, error) +} + +func (f upgradeFactory) Create(registry.ID) (process.Process, *process.Meta, error) { + return f.create() +} + +// An actor spawned with max_steps keeps that cap when it upgrades into a definition that never finishes. +func TestUpgradeKeepsSpawnMaxSteps(t *testing.T) { + executor := newBenchExecutorWithOptions(newMockRegistry(), scheduler.WithWorkers(1)) + executor.Start() + defer executor.Stop() + + upgradeReq := &UpgradeRequest{Source: registry.NewID("test", "next")} + source := mustNewProcess(t, + WithScript(` + local s = 0 + for i = 1, 300 do s = s + i end + coroutine.yield(upgrade_request) + `, "source.lua"), + WithProcessExecutionBudgets(luaapi.ExecutionBudgets{TickBudget: 100, TickBudgetSet: true}), + WithModuleBinder(wrapBinder(func(l *lua.LState) { l.SetGlobal("upgrade_request", upgradeReq) })), + ) + + appCtx := ctxapi.WithAppContext(context.Background(), ctxapi.NewAppContext()) + process.WithFactory(appCtx, upgradeFactory{create: func() (process.Process, *process.Meta, error) { + next, err := NewProcess(WithScript(`while true do end`, "next.lua"), + WithProcessExecutionBudgets(luaapi.ExecutionBudgets{TickBudget: 100, TickBudgetSet: true})) + if err != nil { + return nil, nil, err + } + return &mockProcess{Process: next}, &process.Meta{}, nil + }}) + + ctx, cancel := context.WithTimeout(appCtx, 10*time.Second) + defer cancel() + frame, fc := ctxapi.OpenFrameContext(ctx) + if err := runtime.SetFrameID(frame, registry.NewID("test", "source")); err != nil { + t.Fatal(err) + } + if err := fc.Set(runtime.FrameLifecycleOptionsKey, attrs.Attributes(attrs.Bag{"max_steps": int64(20)})); err != nil { + t.Fatal(err) + } + + result, err := executor.Execute(frame, newTestPID("upgrader"), &mockProcess{Process: source}, "", nil) + if err != nil { + t.Fatal(err) + } + if !errors.Is(result.Error, process.ErrStepLimitExceeded) { + t.Fatalf("expected the upgraded actor to fail with ErrStepLimitExceeded, got %v", result.Error) + } +} + +// A coroutine that keeps hitting its budget does not starve the actor's other +// coroutines: preempted tasks queue behind the ones still waiting. +func TestProcessPreemptedCoroutineDoesNotStarveOthers(t *testing.T) { + proc := initPreemptProcess(t, ` + coroutine.spawn(function() while true do end end) + local s = 0 + for i = 1, 20000 do s = s + i end + return s + `, 200) + + v, preempts := stepPreempted(t, proc, 5000) + if lua.LVAsNumber(v) != 200010000 { + t.Fatalf("expected 200010000, got %v", v) + } + if preempts == 0 { + t.Fatal("expected preemption") + } +} + +func BenchmarkPreemptedStepManyCoroutines(b *testing.B) { + for _, n := range []int{10, 100, 1000} { + b.Run(fmt.Sprintf("tasks=%d", n), func(b *testing.B) { + proc := mustNewProcess(b, WithScript(fmt.Sprintf(` + for _ = 1, %d do + coroutine.spawn(function() while true do for _ = 1, 1000 do end coroutine.yield() end end) + end + while true do coroutine.yield() end + `, n), "many.lua"), WithProcessExecutionBudgets(luaapi.ExecutionBudgets{TickBudget: 64, TickBudgetSet: true})) + proc.EnablePreemption() + ctx, _ := ctxapi.OpenFrameContext(context.Background()) + if err := proc.Init(ctx, "", nil); err != nil { + b.Fatal(err) + } + defer proc.Close() + var output process.StepOutput + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + output.Reset() + if err := proc.Step(nil, &output); err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/runtime/lua/engine/process.go b/runtime/lua/engine/process.go index 9f3ff0aa6..b0a234292 100644 --- a/runtime/lua/engine/process.go +++ b/runtime/lua/engine/process.go @@ -19,6 +19,7 @@ import ( "github.com/wippyai/runtime/api/process" "github.com/wippyai/runtime/api/registry" "github.com/wippyai/runtime/api/relay" + "github.com/wippyai/runtime/api/runtime" luaapi "github.com/wippyai/runtime/api/runtime/lua" "github.com/wippyai/runtime/api/runtime/resource" "github.com/wippyai/runtime/api/topology" @@ -31,6 +32,17 @@ var processPool = sync.Pool{ New: func() any { return nil }, } +// DefaultActorTickBudget is the tick budget of a preemptible actor whose +// tick_budget option is unset or zero. +const DefaultActorTickBudget int64 = 512 + +// WithProcessExecutionBudgets sets the process entry's execution options. +func WithProcessExecutionBudgets(budgets luaapi.ExecutionBudgets) ProcessOption { + return func(p *Process) { + p.entryBudgets = budgets + } +} + // ProcessOption configures a Process. type ProcessOption func(*Process) @@ -89,10 +101,13 @@ type Process struct { messageQueueLimits map[string]int64 messageQueueItemLimits map[string]int mainTask *Task - upgradeRequest *UpgradeRequest - proto *lua.FunctionProto - queue *TaskQueue - factory *Factory + // preempted is the task the VM suspended when the step's tick budget ran + // out; it re-enters the run queue behind the tasks still waiting. + preempted *Task + upgradeRequest *UpgradeRequest + proto *lua.FunctionProto + queue *TaskQueue + factory *Factory // pendingOutdated holds the single coalesced OUTDATED event awaiting // delivery to an upgradable process's events channel. Nil when none pending. pendingOutdated *topology.OutdatedEvent @@ -103,8 +118,11 @@ type Process struct { // stalledChans records channels that could not accept a message during the // current flush. Later messages for the same channel, including terminals, // remain queued so delivery order cannot be inverted. - stalledChans map[*Channel]struct{} - exported map[string]*lua.LFunction + stalledChans map[*Channel]struct{} + exported map[string]*lua.LFunction + // exportFn is the Go function the module bootstrap calls to cache the + // requested method; created on first use. + exportFn *lua.LFunction messageQueueDiscarded map[string]struct{} messageQueueOverflowed map[string]struct{} channelQueue *TaskQueue @@ -123,6 +141,11 @@ type Process struct { outTasks []*Task threads []*Task yieldSeq uint64 + // entryBudgets are the execution options of the process entry; spawn + // options override them at Init. + entryBudgets luaapi.ExecutionBudgets + budgets luaapi.ExecutionBudgets + steps uint64 // epoch is the monotonic incarnation counter. Incremented on every // Init / clearExecution / Close drain and on Abort. Producers stamp // every SubscriptionFrame with the epoch they were registered under; @@ -133,6 +156,9 @@ type Process struct { messageBatchActive bool trapLinks bool upgradable bool + // preemptive is set by schedulers that run the process again after it + // reports StepPreempted. + preemptive bool } // queuedMessage stores a message waiting to be delivered @@ -544,6 +570,13 @@ func (p *Process) Init(ctx context.Context, method string, input payload.Payload p.mainTask = nil p.pendingYields = nil + budgets, err := p.executionBudgets(ctx) + if err != nil { + return err + } + p.budgets = budgets + p.steps = 0 + // Set context for this execution p.ctx = ctx p.state.SetContext(ctx) @@ -592,35 +625,36 @@ func (p *Process) Init(ctx context.Context, method string, input payload.Payload // Determine which function to execute var fn *lua.LFunction - - // If method specified, try to use cached function or extract from module - if method != "" { - if p.exported != nil { - if cached, ok := p.exported[method]; ok { - fn = cached - } + var bootstrap []lua.LValue + + inits := initializersOf(p.state) + pending := inits.pending() + if method != "" && !pending { + fn = p.exported[method] + } + if fn == nil && (method != "" || pending) { + // The module chunk and the initializers it depends on run inside the + // main task so their execution is scheduled and budgeted like any + // other Lua code. + chunk, err := p.loadMain() + if err != nil { + return err } - - // No cached function - need to run script to get module and extract method - if fn == nil { - if err := p.extractMethod(method); err != nil { - return err - } - fn = p.exported[method] + fn = p.state.LoadProto(entryBootstrapProto) + advance := p.state.NewFunction(func(*lua.LState) int { return 0 }) + if inits != nil { + inits.begin() + advance = p.state.NewFunction(inits.advance) } - } else { - // No method - run the script directly (legacy behavior) - - if p.proto != nil { - fn = p.state.LoadProto(p.proto) - } else if p.script != "" { - var err error - fn, err = p.state.Load(strings.NewReader(p.script), p.scriptName) - if err != nil { - return runtimelua.NewLoadScriptError(err) - } - } else { - return luaapi.ErrNoScriptOrProto + export, name := lua.LValue(lua.LFalse), lua.LValue(lua.LFalse) + if method != "" { + export, name = p.exportFunction(), lua.LString(method) + } + bootstrap = []lua.LValue{chunk, advance, export, name} + } else if fn == nil { + var err error + if fn, err = p.loadMain(); err != nil { + return err } } @@ -628,8 +662,9 @@ func (p *Process) Init(ctx context.Context, method string, input payload.Payload p.mainTask = p.createTask(fn) // Convert input payloads to Lua values as arguments - if len(input) > 0 { - args := make([]lua.LValue, 0, len(input)) + if len(input) > 0 || bootstrap != nil { + args := make([]lua.LValue, 0, len(bootstrap)+len(input)) + args = append(args, bootstrap...) for _, pl := range input { args = append(args, transcodeToLua(ctx, pl)) } @@ -639,65 +674,81 @@ func (p *Process) Init(ctx context.Context, method string, input payload.Payload return nil } -// extractMethod runs the script to get module table and extracts the method function. -func (p *Process) extractMethod(method string) error { - // Load script function - var scriptFn *lua.LFunction +// entryBootstrapProto runs the deferred initializers, then the entry chunk. It +// receives the chunk function, the initializer iterator, the export function +// and the method name, followed by the process input. A false method runs the +// chunk itself with the input; otherwise the chunk returns the module the +// method is exported from. The code uses no global, so guest code cannot +// change it. +var entryBootstrapProto = mustCompileBootstrap(` +return (function(chunk, advance, export, method, ...) + local init = advance() + while init do + init = advance(init()) + end + if not method then + return chunk(...) + end + return export(method, (chunk()))(...) +end)(...) +`, "=entry_bootstrap") + +func mustCompileBootstrap(source, name string) *lua.FunctionProto { + proto, err := lua.CompileString(source, name) + if err != nil { + panic(err) + } + return proto +} - if p.proto != nil { - scriptFn = p.state.LoadProto(p.proto) - } else if p.script != "" { - var err error - scriptFn, err = p.state.Load(strings.NewReader(p.script), p.scriptName) +// loadMain loads the process script as a function without running it. +func (p *Process) loadMain() (*lua.LFunction, error) { + switch { + case p.proto != nil: + return p.state.LoadProto(p.proto), nil + case p.script != "": + fn, err := p.state.Load(strings.NewReader(p.script), p.scriptName) if err != nil { - return runtimelua.NewLoadScriptError(err) + return nil, runtimelua.NewLoadScriptError(err) } - } else { - return luaapi.ErrNoScriptOrProto + return fn, nil + default: + return nil, luaapi.ErrNoScriptOrProto } +} - // Run script synchronously to get module table - if err := p.state.CallByParam(lua.P{ - Fn: scriptFn, - NRet: 1, - Protect: true, - }); err != nil { - var raised *lua.Error - if errors.As(err, &raised) { - return toAPIError(raised) - } - return runtimelua.NewExecuteScriptError(err) +// exportFunction returns the Go function that resolves a method from the value +// a module chunk returned and caches it for later executions of this process. +func (p *Process) exportFunction() *lua.LFunction { + if p.exportFn == nil { + p.exportFn = p.state.NewFunction(p.exportMethod) } + return p.exportFn +} - // Get return value - ret := p.state.Get(-1) - p.state.Pop(1) +// exportMethod takes (method, moduleValue) and returns the method function. +// The module value is either the function itself or a table holding it. +func (p *Process) exportMethod(l *lua.LState) int { + method := l.CheckString(1) - // Initialize exported map - if p.exported == nil { - p.exported = make(map[string]*lua.LFunction) - } - - // Extract function from return value var fn *lua.LFunction - switch v := ret.(type) { + switch v := l.Get(2).(type) { case *lua.LFunction: - // Script returned function directly fn = v case *lua.LTable: - // Script returned module table - extract method by name - val := v.RawGetString(method) - if val.Type() == lua.LTFunction { - fn = val.(*lua.LFunction) - } + fn, _ = v.RawGetString(method).(*lua.LFunction) } - if fn == nil { - return runtimelua.NewMethodNotFoundError(method) + l.Error(lua.WrapError(runtimelua.NewMethodNotFoundError(method), "").WithKind(lua.NotFound).WithRetryable(false), 1) + return 0 } + if p.exported == nil { + p.exported = make(map[string]*lua.LFunction) + } p.exported[method] = fn - return nil + l.Push(fn) + return 1 } // Step advances the process by one iteration. @@ -741,6 +792,16 @@ func (p *Process) Step(events []process.Event, out *process.StepOutput) error { defer p.finishMessageBatch(base) } + // Events are consumed before the step is charged, so a step that exceeds + // its limit leaves their retention leases with the mailbox, which releases + // them when the process closes. + if err := p.chargeStep(); err != nil { + p.clearExecution() + out.Done(nil) + return err + } + p.state.SetTickBudget(p.stepTickBudget()) + // Process in a loop until stable (no new subscriptions trigger channel work) var externalTasks []*Task var err error @@ -785,6 +846,17 @@ func (p *Process) Step(events []process.Event, out *process.StepOutput) error { // with only the unhandled tasks (subscription yields were removed) p.externalTasks = externalTasks + // An exhausted tick budget ends the step; tasks still queued run in + // the next one. Messages held back for a receiver that parked in this + // step are delivered first, so the step does not report idle while a + // queued message matches a parked receiver. + if p.state.TickBudget() == 0 { + if p.subs != nil { + p.flushMessageQueue(p.subs) + } + break + } + // Continue looping if subscriptions were handled (may have added tasks) // or if queue has tasks to process. // @@ -804,6 +876,14 @@ func (p *Process) Step(events []process.Event, out *process.StepOutput) error { } } + // The VM-preempted task runs again behind the tasks that did not get a + // turn in this step. + if p.preempted != nil { + p.queue.Push(p.preempted) + p.preempted = nil + } + preempted := p.state.TickBudget() == 0 && !p.queue.IsEmpty() + // Check for upgrade request if p.upgradeRequest != nil { req := p.upgradeRequest @@ -825,11 +905,6 @@ func (p *Process) Step(events []process.Event, out *process.StepOutput) error { return toAPIError(execErr) } - // Initialize pendingYields map if needed - if p.pendingYields == nil { - p.pendingYields = make(map[uint64]*Task, 4) - } - // Convert external yields to commands yieldCount := 0 for _, task := range externalTasks { @@ -841,6 +916,9 @@ func (p *Process) Step(events []process.Event, out *process.StepOutput) error { // Check for scheduler commands in yielded values cmd := p.yieldToCommand(task) if cmd != nil { + if p.pendingYields == nil { + p.pendingYields = make(map[uint64]*Task, 4) + } p.yieldSeq++ p.pendingYields[p.yieldSeq] = task out.Yield(cmd, p.yieldSeq) @@ -852,6 +930,8 @@ func (p *Process) Step(events []process.Event, out *process.StepOutput) error { // Determine status switch { + case preempted: + out.Preempt() case yieldCount == 0 && !p.queue.IsEmpty(): out.Continue() case yieldCount == 0 && len(p.threads) > 0: @@ -877,6 +957,50 @@ func (p *Process) Step(events []process.Event, out *process.StepOutput) error { return nil } +// EnablePreemption implements process.Preemptible. Each step may then run +// the actor's tick budget of VM ticks (backward jumps, loop iterations and +// calls) before the running coroutine is preempted at a safepoint and the +// step reports StepPreempted. +func (p *Process) EnablePreemption() { + p.preemptive = true +} + +// stepTickBudget returns the VM tick budget for one step; -1 is unlimited. +func (p *Process) stepTickBudget() int64 { + if !p.preemptive { + return -1 + } + switch budget := p.budgets.TickBudget; { + case budget < 0: + return -1 + case budget == 0: + return DefaultActorTickBudget + default: + return budget + } +} + +// chargeStep counts a step and fails it when the count exceeds a non-zero +// max_steps option. +func (p *Process) chargeStep() error { + p.steps++ + if p.budgets.MaxSteps != 0 && p.steps > p.budgets.MaxSteps { + return process.ErrStepLimitExceeded + } + return nil +} + +// StepsUsed implements process.StepAccounted. +func (p *Process) StepsUsed() uint64 { + return p.steps +} + +// ResumeStepCount implements process.StepAccounted. The count continues from +// used, so max_steps bounds the actor across upgrades. +func (p *Process) ResumeStepCount(used uint64) { + p.steps = used +} + // State returns the underlying Lua state. func (p *Process) State() *lua.LState { return p.state @@ -912,12 +1036,8 @@ func (p *Process) processChannelYields() ([]*Task, error) { channels := p.channels channelQueue := p.ChannelQueue() - // Transfer tasks from process queue to channel queue - for _, task := range p.queue.Drain() { - channelQueue.Push(task) - } - - // Process all queued tasks + // Tasks already in the process queue run first; tasks that channel + // operations wake follow through the channel queue. boot := true for !channelQueue.IsEmpty() || boot { boot = false @@ -1718,13 +1838,13 @@ func (p *Process) vmStep(tasks ...*Task) ([]*Task, error) { // Reuse yield buffer p.yieldBuf = p.yieldBuf[:0] - for !p.queue.IsEmpty() { + for !p.queue.IsEmpty() && p.state.TickBudget() != 0 { task := p.queue.Pop() if task == nil { continue } - if task.State != lua.ResumeYield { + if task.State != lua.ResumeYield && task.State != lua.ResumePreempted { continue } @@ -1784,6 +1904,8 @@ func (p *Process) vmStep(tasks ...*Task) ([]*Task, error) { return nil, nil } p.removeTask(task) + case lua.ResumePreempted: + p.preempted = task case lua.ResumeError: p.removeTask(task) } @@ -1829,6 +1951,7 @@ func (p *Process) killAllThreads() { } p.threads = p.threads[:0] p.queue.Drain() + p.preempted = nil } // handleSpawnRequest checks if yielded values contain a SpawnRequest and handles it. @@ -1918,6 +2041,7 @@ func (p *Process) Close() { p.yieldBuf = p.yieldBuf[:0] p.externalTasks = p.externalTasks[:0] p.outTasks = p.outTasks[:0] + p.preempted = nil p.clearMessageQueue() // Clear all references @@ -1931,6 +2055,7 @@ func (p *Process) Close() { p.factory = nil p.pendingYields = nil p.exported = nil + p.exportFn = nil p.channelQueue = nil p.channels = nil p.subs = nil @@ -1950,11 +2075,30 @@ func (p *Process) Close() { p.upgradable = false p.pendingOutdated = nil p.linkDownError = nil + p.entryBudgets = luaapi.ExecutionBudgets{} + p.budgets = luaapi.ExecutionBudgets{} + p.steps = 0 + p.preemptive = false // Return to pool processPool.Put(p) } +// executionBudgets resolves the execution options for this execution: the +// entry's options, overridden by the spawn options in the frame context. +func (p *Process) executionBudgets(ctx context.Context) (luaapi.ExecutionBudgets, error) { + budgets := p.entryBudgets + options, ok := runtime.GetFrameLifecycleOptions(ctx).(attrs.Attributes) + if !ok || options == nil { + return budgets, nil + } + spawn, err := luaapi.ExecutionBudgetsFromOptions(options, "process spawn options") + if err != nil { + return budgets, err + } + return budgets.Override(spawn), nil +} + // SyncExecute runs the script directly without coroutines or scheduler. // This is a fast path for simple synchronous calls that don't need yields. // The Lua state must be initialized via Start first. @@ -1965,6 +2109,10 @@ func (p *Process) SyncExecute(ctx context.Context, args ...lua.LValue) (lua.LVal p.state.SetContext(ctx) + if err := initializersOf(p.state).runSync(p.state); err != nil { + return lua.LNil, toAPIError(err) + } + // Load function from proto fn := p.state.LoadProto(p.proto) @@ -2045,6 +2193,7 @@ func (p *Process) clearExecution() { // Drain queue p.queue.Drain() + p.preempted = nil // Clear main task reference and result p.mainTask = nil diff --git a/runtime/lua/engine/process_factory.go b/runtime/lua/engine/process_factory.go index a66120843..ee9e4b491 100644 --- a/runtime/lua/engine/process_factory.go +++ b/runtime/lua/engine/process_factory.go @@ -49,6 +49,7 @@ type processConfig struct { excludeModules []string extraModules []*luaapi.ModuleDef buildMode code.AccessMode + budgets luaapi.ExecutionBudgets } func newProcessConfig() *processConfig { @@ -107,6 +108,14 @@ func WithModules(mods ...*luaapi.ModuleDef) FactoryOption { } } +// WithExecutionBudgets sets the execution options (tick_budget, max_steps) +// of the processes the factory creates. +func WithExecutionBudgets(budgets luaapi.ExecutionBudgets) FactoryOption { + return func(c *processConfig) { + c.budgets = budgets + } +} + // WithFilter sets a custom filter function. // Return (true, nil) to include, (false, nil) to exclude, (false, err) to fail. func WithFilter(fn func(name string, classes []string) (bool, error)) FactoryOption { @@ -152,6 +161,7 @@ func (f *ProcessFactory) CreateFactory(id registry.ID, opts ...FactoryOption) (p factoryCfg := FactoryConfig{ Proto: compiled.Main, ModuleBinders: binders, + Budgets: cfg.budgets, } factory := NewFactory(factoryCfg) @@ -211,6 +221,7 @@ func (f *ProcessFactory) isolationBinder( base := l.Get(lua.GlobalsIndex).(*lua.LTable) valueByNode := make(map[registry.ID]lua.LValue, len(compiled.Dependencies)) processed := make(map[registry.ID]struct{}, len(compiled.Dependencies)) + var seals []func() // Dependencies are ordered deepest-first, so a library's imports are // resolved before the library itself runs. @@ -255,21 +266,30 @@ func (f *ProcessFactory) isolationBinder( } if dep.Proto != nil { - env := buildChunkEnv(l, base, compiled.Imports[id], valueByNode) + env, seal := buildChunkEnv(l, base, compiled.Imports[id], valueByNode) + seals = append(seals, seal) fn := l.LoadProto(dep.Proto) fn.Env = env - l.Push(fn) - if err := l.PCall(0, 1, nil); err != nil { - return fmt.Errorf("failed to load dependency %s: %w", dep.Name, toAPIError(err)) - } - valueByNode[id] = l.Get(-1) - l.Pop(1) + DeferInitializer(l, Initializer{ + Fn: fn, + Done: func(result lua.LValue) { valueByNode[id] = result }, + }) } } // The entrypoint runs under its own scoped environment. Assigning it as // the state environment makes LoadProto stamp the main chunk with it. - l.Env = buildChunkEnv(l, base, compiled.Imports[compiled.MainID], valueByNode) + env, seal := buildChunkEnv(l, base, compiled.Imports[compiled.MainID], valueByNode) + l.Env = env + seals = append(seals, seal) + + // Once every library has initialized, each environment fixes which + // aliases are imports against the globals as they then are. + OnInitialized(l, func() { + for _, seal := range seals { + seal() + } + }) return nil } } @@ -279,30 +299,60 @@ func (f *ProcessFactory) isolationBinder( // through a scoped require) and falls back to the shared safe base for standard // globals via its metatable. // -// A runtime-installed DSL global may temporarily shadow an import alias when -// that global did not exist in the base environment at chunk creation time. This -// preserves the old shared-_G migration DSL behavior without leaking declared -// imports into _G. Undeclared modules resolve to nil and require fails closed. -func buildChunkEnv(l *lua.LState, base *lua.LTable, imports []code.Import, valueByNode map[registry.ID]lua.LValue) *lua.LTable { +// An alias is an import when the base environment holds a value under that +// name at the moment the environment is sealed: after library initialization +// completes, or at the first lookup if that comes earlier. A global installed +// under an alias name before sealing is shadowed by the import; one installed +// afterwards, such as the migration DSL's helpers, stays visible through the +// base without leaking declared imports into _G. Undeclared modules resolve to +// nil and require fails closed. +func buildChunkEnv(l *lua.LState, base *lua.LTable, imports []code.Import, valueByNode map[registry.ID]lua.LValue) (*lua.LTable, func()) { env := l.NewTable() - resolved := make(map[string]lua.LValue, len(imports)) - baseHadAlias := make(map[string]bool, len(imports)) + // Library values appear in valueByNode as their initializers complete, so + // imports resolve at lookup time. Imports sharing an alias resolve to the + // last one that has a value. + aliasIDs := make(map[string][]registry.ID, len(imports)) for _, imp := range imports { - v, ok := valueByNode[imp.ID] - if !ok { - continue - } alias := imp.Alias if alias == "" { alias = imp.ID.Name } - resolved[alias] = v - baseHadAlias[alias] = base.RawGetString(alias) != lua.LNil + aliasIDs[alias] = append(aliasIDs[alias], imp.ID) + } + + // An alias that already exists in the base is an import: the declared + // import wins over it, so a library's explicit _G export under an alias + // name does not override the import, while helpers installed later resolve + // through the base. The returned seal fixes the set against the current + // globals; it is called when library initialization completes. Lookups + // before that, which only library top-level code makes, fix it on first + // use, with the chunk's own dependencies already run. + var baseHadAlias map[string]bool + seal := func() { + baseHadAlias = make(map[string]bool, len(aliasIDs)) + for alias := range aliasIDs { + baseHadAlias[alias] = base.RawGetString(alias) != lua.LNil + } + } + snapshot := func() { + if baseHadAlias == nil { + seal() + } + } + resolve := func(alias string) (lua.LValue, bool) { + ids := aliasIDs[alias] + for i := len(ids) - 1; i >= 0; i-- { + if v, ok := valueByNode[ids[i]]; ok { + return v, true + } + } + return nil, false } mt := l.NewTable() mt.RawSetString("__index", l.NewFunction(func(s *lua.LState) int { + snapshot() name := s.CheckString(2) baseValue := base.RawGetString(name) if baseValue != lua.LNil && !baseHadAlias[name] { @@ -312,7 +362,7 @@ func buildChunkEnv(l *lua.LState, base *lua.LTable, imports []code.Import, value s.Push(baseValue) return 1 } - if v, ok := resolved[name]; ok { + if v, ok := resolve(name); ok { s.Push(v) return 1 } @@ -326,8 +376,9 @@ func buildChunkEnv(l *lua.LState, base *lua.LTable, imports []code.Import, value l.SetMetatable(env, mt) env.RawSetString("require", l.NewFunction(func(s *lua.LState) int { + snapshot() name := s.CheckString(1) - if v, ok := resolved[name]; ok { + if v, ok := resolve(name); ok { s.Push(v) return 1 } @@ -341,7 +392,7 @@ func buildChunkEnv(l *lua.LState, base *lua.LTable, imports []code.Import, value s.RaiseError("module '%s' not found", name) return 0 })) - return env + return env, seal } // Helper functions @@ -372,6 +423,7 @@ type FactoryConfig struct { Script string ScriptName string ModuleBinders []ModuleBinder + Budgets luaapi.ExecutionBudgets } // Factory creates Lua processes with shared configuration. @@ -382,6 +434,7 @@ type Factory struct { script string scriptName string moduleBinders []ModuleBinder + budgets luaapi.ExecutionBudgets } // NewFactory creates a ProcessFactory for Lua processes. @@ -393,6 +446,7 @@ func NewFactory(cfg FactoryConfig) process.FactoryFunc { scriptName: cfg.ScriptName, moduleBinders: cfg.ModuleBinders, stateOpts: cfg.StateOptions, + budgets: cfg.Budgets, } return f.Create } @@ -405,11 +459,12 @@ func (f *Factory) Create() (process.Process, error) { } proc := &Process{ - threads: make([]*Task, 0, 4), - queue: NewTaskQueue(), - yieldBuf: make([]*Task, 0, 4), - factory: f, - state: state, + threads: make([]*Task, 0, 4), + queue: NewTaskQueue(), + yieldBuf: make([]*Task, 0, 4), + factory: f, + state: state, + entryBudgets: f.budgets, } if f.proto != nil { diff --git a/runtime/lua/engine/process_factory_test.go b/runtime/lua/engine/process_factory_test.go index 2cdedca01..e05fb706e 100644 --- a/runtime/lua/engine/process_factory_test.go +++ b/runtime/lua/engine/process_factory_test.go @@ -3,10 +3,14 @@ package engine import ( + "context" "testing" + "time" lua "github.com/wippyai/go-lua" + ctxapi "github.com/wippyai/runtime/api/context" apierror "github.com/wippyai/runtime/api/error" + "github.com/wippyai/runtime/api/process" "github.com/wippyai/runtime/api/registry" luaapi "github.com/wippyai/runtime/api/runtime/lua" "github.com/wippyai/runtime/runtime/lua/code" @@ -259,9 +263,6 @@ func TestNewProcessFactory(t *testing.T) { } func TestDependencyRaisePreservesKind(t *testing.T) { - l := lua.NewState() - defer l.Close() - lua.OpenErrors(l) id := registry.NewID("test", "dependency") compiled := &code.CompiledMain{ Dependencies: []code.CompiledProto{{ @@ -270,14 +271,133 @@ func TestDependencyRaisePreservesKind(t *testing.T) { Proto: compileTestProto(t, `error(errors.new({message="bad dependency", kind=errors.INVALID, retryable=false}))`), }}, } - pf := NewProcessFactory(nil) - binder := pf.isolationBinder(compiled, newProcessConfig(), nil, nil, nil, nil) - err := binder(l) + proc := newDependencyProcess(t, compiled, `return { main = function() end }`, 0) + if err := proc.Init(frameContext(), "main", nil); err != nil { + t.Fatal(err) + } + var output process.StepOutput + err := proc.Step(nil, &output) if err == nil || apierror.BuildChain(err).Root().Kind != string(apierror.Invalid) { t.Fatalf("dependency error = %v", err) } } +// Library chunks run in dependency order under per-chunk environments, inside +// the first step. +func TestDependenciesRunInFirstStepWithScopedEnvironments(t *testing.T) { + baseID := registry.NewID("test", "base") + midID := registry.NewID("test", "mid") + mainID := registry.NewID("test", "main") + compiled := &code.CompiledMain{ + MainID: mainID, + Imports: map[registry.ID][]code.Import{ + midID: {{ID: baseID, Alias: "b"}}, + mainID: {{ID: midID, Alias: "m"}}, + }, + Dependencies: []code.CompiledProto{ + {Name: "base", Node: &code.Node{ID: baseID}, Proto: compileTestProto(t, `ran = (ran or 0) + 1 return { v = 1 }`)}, + {Name: "mid", Node: &code.Node{ID: midID}, Proto: compileTestProto(t, `return { v = b.v + 1, via_require = require("b").v }`)}, + }, + } + proc := newDependencyProcess(t, compiled, ` + return { main = function() + return m.v * 100 + require("m").via_require * 10 + (b == nil and 1 or 0) + end } + `, 0) + state := proc.State() + if state.GetGlobal("ran") != lua.LNil { + t.Fatal("library code must not run before the first step") + } + if err := proc.Init(frameContext(), "main", nil); err != nil { + t.Fatal(err) + } + var output process.StepOutput + if err := proc.Step(nil, &output); err != nil { + t.Fatal(err) + } + // m.v = 2, require("m").via_require = 1, b hidden from main = 1. + if v, _ := output.Result().Data().(lua.LValue); lua.LVAsNumber(v) != 211 { + t.Fatalf("expected 211, got %v", output.Result().Data()) + } +} + +// A library that never returns is preempted instead of blocking Init, and a +// later execution of a process with finished libraries does not run them again. +func TestDependencyLoopIsPreempted(t *testing.T) { + id := registry.NewID("test", "spin") + compiled := &code.CompiledMain{ + Dependencies: []code.CompiledProto{{Name: "spin", Node: &code.Node{ID: id}, Proto: compileTestProto(t, `while true do end`)}}, + } + proc := newDependencyProcess(t, compiled, `return { main = function() end }`, 100) + proc.EnablePreemption() + + inited := make(chan error, 1) + go func() { inited <- proc.Init(frameContext(), "main", nil) }() + select { + case err := <-inited: + if err != nil { + t.Fatal(err) + } + case <-time.After(5 * time.Second): + t.Fatal("Init blocked on library code") + } + var output process.StepOutput + if err := proc.Step(nil, &output); err != nil { + t.Fatal(err) + } + if output.Status() != process.StepPreempted { + t.Fatalf("expected the library loop to be preempted, got %v", output.Status()) + } +} + +func TestDependenciesRunOncePerProcess(t *testing.T) { + id := registry.NewID("test", "counter") + mainID := registry.NewID("test", "main") + compiled := &code.CompiledMain{ + MainID: mainID, + Imports: map[registry.ID][]code.Import{mainID: {{ID: id, Alias: "c"}}}, + Dependencies: []code.CompiledProto{{Name: "counter", Node: &code.Node{ID: id}, Proto: compileTestProto(t, + `runs = (runs or 0) + 1 return { runs = runs }`)}}, + } + proc := newDependencyProcess(t, compiled, `return { main = function() return c.runs end }`, 0) + for i := 0; i < 3; i++ { + if err := proc.Init(frameContext(), "main", nil); err != nil { + t.Fatal(err) + } + var output process.StepOutput + if err := proc.Step(nil, &output); err != nil { + t.Fatal(err) + } + if v, _ := output.Result().Data().(lua.LValue); lua.LVAsNumber(v) != 1 { + t.Fatalf("execution %d: expected the library to have run once, got %v", i, output.Result().Data()) + } + } +} + +func frameContext() context.Context { + ctx, _ := ctxapi.OpenFrameContext(context.Background()) + return ctx +} + +// newDependencyProcess builds a process over script whose state is bound +// through the factory's isolation binder for compiled. +func newDependencyProcess(t *testing.T, compiled *code.CompiledMain, script string, tickBudget int64) *Process { + t.Helper() + binder := NewProcessFactory(nil).isolationBinder(compiled, newProcessConfig(), nil, nil, nil, nil) + budgets := luaapi.ExecutionBudgets{} + if tickBudget != 0 { + budgets = luaapi.ExecutionBudgets{TickBudget: tickBudget, TickBudgetSet: true} + } + proc := mustNewProcess(t, + WithModuleBinder(wrapBinder(func(l *lua.LState) { lua.OpenErrors(l) })), + WithModuleBinder(binder), + WithScript(script, "main.lua"), + WithProcessExecutionBudgets(budgets), + ) + t.Cleanup(proc.Close) + return proc +} + // --- helpers --- func compileTestProto(t *testing.T, source string) *lua.FunctionProto { @@ -289,3 +409,124 @@ func compileTestProto(t *testing.T, source string) *lua.FunctionProto { require.NoError(t, err) return fn.Proto } + +// A library that assigns _G under the alias it is imported as does not +// override the declared import: the alias resolves the same through the +// environment and through require. +func TestLibraryGlobalExportDoesNotOverrideImportAlias(t *testing.T) { + id := registry.NewID("test", "c") + mainID := registry.NewID("test", "main") + compiled := &code.CompiledMain{ + MainID: mainID, + Imports: map[registry.ID][]code.Import{mainID: {{ID: id, Alias: "c"}}}, + Dependencies: []code.CompiledProto{{Name: "c", Node: &code.Node{ID: id}, Proto: compileTestProto(t, + `_G.c = { v = 99 } _G.other = 7 return { v = 42 }`)}}, + } + proc := newDependencyProcess(t, compiled, ` + return { main = function() return c.v * 1000 + require("c").v + other end } + `, 0) + runProcess(t, proc, 42049) +} + +// A global installed after the environment is in use stays visible even when +// it shares a name with an import alias. +func TestDynamicGlobalOverridesImportAliasAfterInitialization(t *testing.T) { + id := registry.NewID("test", "dsl") + mainID := registry.NewID("test", "main") + compiled := &code.CompiledMain{ + MainID: mainID, + Imports: map[registry.ID][]code.Import{mainID: {{ID: id, Alias: "dsl"}}}, + Dependencies: []code.CompiledProto{{Name: "dsl", Node: &code.Node{ID: id}, Proto: compileTestProto(t, + `return { with = function(fn) _G.dsl = function() return 5 end local r = fn() _G.dsl = nil return r end }`)}}, + } + proc := newDependencyProcess(t, compiled, ` + return { main = function() + return require("dsl").with(function() return dsl() end) + end } + `, 0) + runProcess(t, proc, 5) +} + +// SyncExecute initializes the process's libraries before it runs the chunk. +func TestSyncExecuteRunsPendingInitializers(t *testing.T) { + id := registry.NewID("test", "c") + mainID := registry.NewID("test", "main") + compiled := &code.CompiledMain{ + MainID: mainID, + Imports: map[registry.ID][]code.Import{mainID: {{ID: id, Alias: "c"}}}, + Dependencies: []code.CompiledProto{{Name: "c", Node: &code.Node{ID: id}, Proto: compileTestProto(t, + `_G.runs = (_G.runs or 0) + 1 return { v = 42 }`)}}, + } + binder := NewProcessFactory(nil).isolationBinder(compiled, newProcessConfig(), nil, nil, nil, nil) + proc := mustNewProcess(t, WithModuleBinder(binder), WithProto(compileTestProto(t, `return c.v + runs`))) + t.Cleanup(proc.Close) + for i := 0; i < 2; i++ { + got, err := proc.SyncExecute(context.Background()) + if err != nil { + t.Fatal(err) + } + if lua.LVAsNumber(got) != 43 { + t.Fatalf("run %d: expected 43, got %v", i, got) + } + } +} + +func TestSyncExecuteInitializerErrorKeepsKind(t *testing.T) { + id := registry.NewID("test", "c") + compiled := &code.CompiledMain{ + Dependencies: []code.CompiledProto{{Name: "c", Node: &code.Node{ID: id}, Proto: compileTestProto(t, + `error(errors.new({message="bad dependency", kind=errors.INVALID, retryable=false}))`)}}, + } + binder := NewProcessFactory(nil).isolationBinder(compiled, newProcessConfig(), nil, nil, nil, nil) + proc := mustNewProcess(t, + WithModuleBinder(wrapBinder(func(l *lua.LState) { lua.OpenErrors(l) })), + WithModuleBinder(binder), WithProto(compileTestProto(t, `return 1`))) + t.Cleanup(proc.Close) + _, err := proc.SyncExecute(context.Background()) + if err == nil || apierror.BuildChain(err).Root().Kind != string(apierror.Invalid) { + t.Fatalf("expected an Invalid error from the library, got %v", err) + } +} + +func runProcess(t *testing.T, proc *Process, want float64) { + t.Helper() + if err := proc.Init(frameContext(), "main", nil); err != nil { + t.Fatal(err) + } + var output process.StepOutput + if err := proc.Step(nil, &output); err != nil { + t.Fatal(err) + } + if v, _ := output.Result().Data().(lua.LValue); float64(lua.LVAsNumber(v)) != want { + t.Fatalf("expected %v, got %v", want, output.Result().Data()) + } +} + +// Every chunk environment decides its import aliases when library +// initialization completes: a sibling library's _G export under an alias name +// does not override the import in an earlier library, whether or not that +// library looked up a global while it initialized. +func TestSiblingGlobalExportDoesNotOverrideEarlierLibraryImport(t *testing.T) { + for name, aSource := range map[string]string{ + "no top-level lookup": `return { get = function() return c.v end }`, + "top-level global read": `local _ = tostring return { get = function() return c.v end }`, + } { + t.Run(name, func(t *testing.T) { + aID, bID, cID, mainID := registry.NewID("t", "a"), registry.NewID("t", "b"), registry.NewID("t", "c"), registry.NewID("t", "main") + compiled := &code.CompiledMain{ + MainID: mainID, + Imports: map[registry.ID][]code.Import{ + aID: {{ID: cID, Alias: "c"}}, + mainID: {{ID: aID, Alias: "a"}, {ID: bID, Alias: "b"}}, + }, + Dependencies: []code.CompiledProto{ + {Name: "c", Node: &code.Node{ID: cID}, Proto: compileTestProto(t, `return { v = 42 }`)}, + {Name: "a", Node: &code.Node{ID: aID}, Proto: compileTestProto(t, aSource)}, + {Name: "b", Node: &code.Node{ID: bID}, Proto: compileTestProto(t, `_G.c = { v = 99 } return {}`)}, + }, + } + proc := newDependencyProcess(t, compiled, `return { main = function() return a.get() end }`, 0) + runProcess(t, proc, 42) + }) + } +} diff --git a/runtime/lua/engine/process_isolation_test.go b/runtime/lua/engine/process_isolation_test.go index cef8ecba7..80b2bf4b0 100644 --- a/runtime/lua/engine/process_isolation_test.go +++ b/runtime/lua/engine/process_isolation_test.go @@ -76,6 +76,7 @@ func TestFactory_ImportIsolation_LibDepDoesNotLeak(t *testing.T) { if err != nil { t.Fatalf("factory() failed: %v", err) } + runEntry(t, proc.(*Process)) state := proc.(*Process).State() env := state.Env @@ -158,6 +159,7 @@ func TestFactory_ImportIsolation_OverlayLibrary(t *testing.T) { if err != nil { t.Fatalf("factory() failed: %v", err) } + runEntry(t, proc.(*Process)) state := proc.(*Process).State() env := state.Env @@ -230,6 +232,7 @@ func TestFactory_ImportIsolation_ExplicitGlobalExportIsVisibleWithoutImportLeak( defer procRaw.Close() luaProc := procRaw.(*Process) + runEntry(t, luaProc) state := luaProc.State() if got := state.GetGlobal("public_export"); got.String() != "visible" { @@ -316,3 +319,16 @@ func TestFactory_ImportIsolation_DynamicDSLGlobalOverridesImportAlias(t *testing t.Fatalf("migration DSL leaked into _G after define cleanup: %v", v) } } + +// runEntry runs the process's first step, which initializes its libraries. +func runEntry(t *testing.T, proc *Process) { + t.Helper() + ctx, _ := ctxapi.OpenFrameContext(context.Background()) + if err := proc.Init(ctx, "main", nil); err != nil { + t.Fatalf("Init failed: %v", err) + } + var output process.StepOutput + if err := proc.Step(nil, &output); err != nil { + t.Fatalf("Step failed: %v", err) + } +} diff --git a/runtime/lua/engine/process_test.go b/runtime/lua/engine/process_test.go index 792d7fbe7..6bcdbda3e 100644 --- a/runtime/lua/engine/process_test.go +++ b/runtime/lua/engine/process_test.go @@ -1740,6 +1740,21 @@ func TestProcessInitNoScript(t *testing.T) { } } +// requireMethodNotFound checks that err carries the method-not-found error +// in its chain. +func requireMethodNotFound(t *testing.T, err error) { + t.Helper() + if err == nil { + t.Fatal("expected a method-not-found error") + } + for _, link := range apierror.BuildChain(err).Errors { + if link.Kind == string(apierror.NotFound) { + return + } + } + t.Fatalf("expected a not-found error in the chain, got %v", err) +} + // TestProcessInitInvalidMethod tests Init with non-existent method func TestProcessInitInvalidMethod(t *testing.T) { script := `return {}` @@ -1747,12 +1762,14 @@ func TestProcessInitInvalidMethod(t *testing.T) { proc := mustNewProcess(t, WithScript(script, "test.lua")) ctx, _ := ctxapi.OpenFrameContext(context.Background()) - err := proc.Init(ctx, "nonexistent", nil) - - if err == nil { - t.Fatal("expected error for missing method") + if err := proc.Init(ctx, "nonexistent", nil); err != nil { + t.Fatal(err) } - proc.Close() + defer proc.Close() + + var output process.StepOutput + err := proc.Step(nil, &output) + requireMethodNotFound(t, err) } // TestProcessInitSyntaxError tests Init with malformed Lua @@ -3828,6 +3845,11 @@ func TestProcessExtractMethodWithScript(t *testing.T) { } defer proc.Close() + var output process.StepOutput + if err := proc.Step(nil, &output); err != nil { + t.Fatal(err) + } + // Verify method was extracted if proc.exported == nil || proc.exported["handle"] == nil { t.Fatal("handle method should be extracted") @@ -3840,9 +3862,16 @@ func TestProcessExtractMethodNotFound(t *testing.T) { proc := mustNewProcess(t, WithScript(script, "test.lua")) ctx, _ := ctxapi.OpenFrameContext(context.Background()) - err := proc.Init(ctx, "nonexistent", nil) - if err == nil { - t.Fatal("expected error for missing method") + if err := proc.Init(ctx, "nonexistent", nil); err != nil { + t.Fatal(err) + } + defer proc.Close() + + var output process.StepOutput + err := proc.Step(nil, &output) + requireMethodNotFound(t, err) + if output.Status() != process.StepDone { + t.Fatalf("expected the failed process to be done, got %v", output.Status()) } } @@ -3857,6 +3886,11 @@ func TestProcessExtractMethodScriptReturnsFunction(t *testing.T) { } defer proc.Close() + var output process.StepOutput + if err := proc.Step(nil, &output); err != nil { + t.Fatal(err) + } + // Should work - direct function return gets stored if proc.exported == nil || proc.exported["handle"] == nil { t.Fatal("direct function should be extracted") @@ -3869,8 +3903,13 @@ func TestProcessExtractMethodScriptError(t *testing.T) { proc := mustNewProcess(t, WithScript(script, "test.lua")) ctx, _ := ctxapi.OpenFrameContext(context.Background()) - err := proc.Init(ctx, "handle", nil) - if err == nil { + if err := proc.Init(ctx, "handle", nil); err != nil { + t.Fatal(err) + } + defer proc.Close() + + var output process.StepOutput + if err := proc.Step(nil, &output); err == nil { t.Fatal("expected error from script execution") } } diff --git a/runtime/lua/engine/raised_error_test.go b/runtime/lua/engine/raised_error_test.go index 5903d3d1d..707f2fb3e 100644 --- a/runtime/lua/engine/raised_error_test.go +++ b/runtime/lua/engine/raised_error_test.go @@ -17,10 +17,9 @@ func TestProcessRaisedTypedError(t *testing.T) { for _, tc := range []struct { name string script string - init bool }{ - {"body", `return {main=function() error(errors.new({message="bad declaration", kind=errors.INVALID, retryable=false, details={field="target"}})) end}`, false}, - {"initialization", `error(errors.new({message="bad declaration", kind=errors.INVALID, retryable=false, details={field="target"}}))`, true}, + {"body", `return {main=function() error(errors.new({message="bad declaration", kind=errors.INVALID, retryable=false, details={field="target"}})) end}`}, + {"initialization", `error(errors.new({message="bad declaration", kind=errors.INVALID, retryable=false, details={field="target"}}))`}, } { t.Run(tc.name, func(t *testing.T) { factory := NewFactory(FactoryConfig{Script: tc.script, ScriptName: "raised.lua", ModuleBinders: []ModuleBinder{wrapBinder(func(l *lua.LState) { lua.OpenErrors(l) })}}) @@ -31,16 +30,13 @@ func TestProcessRaisedTypedError(t *testing.T) { proc := p.(*Process) defer proc.Close() ctx, _ := ctxapi.OpenFrameContext(context.Background()) - err = proc.Init(ctx, "main", nil) - if !tc.init { - if err != nil { - t.Fatal(err) - } - var output process.StepOutput - err = proc.Step(nil, &output) - if output.Status() != process.StepDone { - t.Fatalf("status = %v", output.Status()) - } + if err := proc.Init(ctx, "main", nil); err != nil { + t.Fatal(err) + } + var output process.StepOutput + err = proc.Step(nil, &output) + if output.Status() != process.StepDone { + t.Fatalf("status = %v", output.Status()) } var apiErr apierror.Error if !errors.As(err, &apiErr) { diff --git a/runtime/lua/engine/task.go b/runtime/lua/engine/task.go index 02e1ab7cf..c67d02b4d 100644 --- a/runtime/lua/engine/task.go +++ b/runtime/lua/engine/task.go @@ -176,6 +176,12 @@ func (t *Task) Close() { t.fn = nil t.Yielded = nil t.Resumed = nil + // The buffers outlive the task in the pool, so their backing arrays must + // not keep the last execution's values reachable. + clear(t.retBuf[:cap(t.retBuf)]) + t.retBuf = t.retBuf[:0] + clear(t.resumeBuf[:cap(t.resumeBuf)]) + t.resumeBuf = t.resumeBuf[:0] t.State = 0 clear(t.resumeBuf) clear(t.retBuf[:cap(t.retBuf)]) diff --git a/runtime/lua/engine/task_test.go b/runtime/lua/engine/task_test.go index fe1520ee1..3726903d3 100644 --- a/runtime/lua/engine/task_test.go +++ b/runtime/lua/engine/task_test.go @@ -218,3 +218,19 @@ func TestTask_PoolReuse(t *testing.T) { assert.Equal(t, lua.ResumeYield, task2.State) task2.Close() } + +// A closed task keeps no Lua values in its reusable buffers. +func TestTaskCloseClearsReusableBuffers(t *testing.T) { + thread := lua.NewState() + task := NewTask(thread, nil) + task.retBuf = append(task.retBuf, lua.LString("result"), lua.LString("result2")) + task.ResumeWith(lua.LString("resumed"), lua.LString("resumed2")) + task.Close() + + for i, v := range task.retBuf[:cap(task.retBuf)] { + require.Nil(t, v, "retBuf[%d]", i) + } + for i, v := range task.resumeBuf[:cap(task.resumeBuf)] { + require.Nil(t, v, "resumeBuf[%d]", i) + } +} diff --git a/runtime/lua/modules/process/context.go b/runtime/lua/modules/process/context.go index 3da067899..d8fd31638 100644 --- a/runtime/lua/modules/process/context.go +++ b/runtime/lua/modules/process/context.go @@ -14,6 +14,7 @@ import ( "github.com/wippyai/runtime/api/registry" "github.com/wippyai/runtime/api/relay" "github.com/wippyai/runtime/api/runtime" + luaapi "github.com/wippyai/runtime/api/runtime/lua" secapi "github.com/wippyai/runtime/api/security" "github.com/wippyai/runtime/runtime/lua/engine/value" "github.com/wippyai/runtime/runtime/security" @@ -86,6 +87,11 @@ func parseSpawnerOptions(l *lua.LState, idx int) attrs.Bag { options.Set(string(key), value.ToGoAny(v)) }) + if _, err := luaapi.ExecutionBudgetsFromOptions(options, "process spawn options"); err != nil { + l.ArgError(idx, err.Error()) + return nil + } + if net := options.GetString(netapi.OptionKeyNetwork, ""); net != "" { if !security.IsAllowed(l.Context(), "network.select", net, nil) { l.RaiseError("not allowed: network %s", net) diff --git a/runtime/lua/modules/process/spawner_security_test.go b/runtime/lua/modules/process/spawner_security_test.go index 000991e36..8f7fa4bc9 100644 --- a/runtime/lua/modules/process/spawner_security_test.go +++ b/runtime/lua/modules/process/spawner_security_test.go @@ -87,3 +87,15 @@ func TestSpawnerContextPermissionStillRequired(t *testing.T) { require.ErrorContains(t, l.DoString(`process.with_options({})`), "custom options") require.ErrorContains(t, l.DoString(`process.with_context({})`), "custom context") } + +func TestSpawnerOptionsValidateExecutionBudgets(t *testing.T) { + l, _ := spawnerSecurityLua(t, &spawnerTestPolicy{contextAllowed: true}) + require.NoError(t, l.DoString(`result = process.with_options({tick_budget = 2048, max_steps = 10})`)) + spawner := l.GetGlobal("result").(*lua.LUserData).Value.(*Spawner) + budget, _ := spawner.options.Get("tick_budget") + require.EqualValues(t, 2048, budget) + + require.ErrorContains(t, l.DoString(`process.with_options({tick_budget = "fast"})`), `"tick_budget" must be an integer`) + require.ErrorContains(t, l.DoString(`process.with_options({tick_budget = 1.5})`), `"tick_budget" must be an integer`) + require.ErrorContains(t, l.DoString(`process.with_options({max_steps = -1})`), `"max_steps" must be a non-negative integer`) +} diff --git a/system/scheduler/actor/chaos_test.go b/system/scheduler/actor/chaos_test.go new file mode 100644 index 000000000..f610c70eb --- /dev/null +++ b/system/scheduler/actor/chaos_test.go @@ -0,0 +1,963 @@ +// SPDX-License-Identifier: MPL-2.0 + +package actor + +import ( + "context" + "errors" + "fmt" + "math/rand" + "os" + goruntime "runtime" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + ctxapi "github.com/wippyai/runtime/api/context" + "github.com/wippyai/runtime/api/dispatcher" + "github.com/wippyai/runtime/api/payload" + pidapi "github.com/wippyai/runtime/api/pid" + "github.com/wippyai/runtime/api/process" + "github.com/wippyai/runtime/api/registry" + "github.com/wippyai/runtime/api/relay" + apiruntime "github.com/wippyai/runtime/api/runtime" + "github.com/wippyai/runtime/api/topology" + sysprocess "github.com/wippyai/runtime/system/process" + "github.com/wippyai/runtime/system/scheduler" +) + +// Chaos configuration. Defaults keep a normal run fast; the environment +// selects longer runs and replays a failing seed: +// +// WIPPY_CHAOS_SEED first seed (default 1) +// WIPPY_CHAOS_RUNS number of consecutive seeds (default 4, 200 with WIPPY_CHAOS_LONG) +// WIPPY_CHAOS_PROCS processes per run (default 120, 600 with WIPPY_CHAOS_LONG) +// WIPPY_CHAOS_LONG any non-empty value selects the long defaults +func chaosEnvInt(name string, short, long int) int { + def := short + if os.Getenv("WIPPY_CHAOS_LONG") != "" { + def = long + } + if v := os.Getenv(name); v != "" { + if n, err := strconv.Atoi(v); err == nil { + return n + } + } + return def +} + +const cmdChaosAsync dispatcher.CommandID = 100 + +// cmdChaosUnregistered has no handler, so a yield of it completes with an +// unknown-command error. +const cmdChaosUnregistered dispatcher.CommandID = 101 + +var ( + errChaosFail = errors.New("chaos: planned failure") + errChaosHandler = errors.New("chaos: handler failure") + errChaosCancel = errors.New("chaos: cancelled") + errChaosStart = errors.New("chaos: start rejected") +) + +type chaosAsyncMode uint8 + +const ( + asyncLater chaosAsyncMode = iota // completes from another goroutine after a delay + asyncInline // completes inside Handle + asyncHandleErr // Handle returns an error + asyncLaterError // completes with an error after a delay + asyncUnknown // no handler +) + +type chaosAsyncCmd struct { + token chaosToken + delay time.Duration + mode chaosAsyncMode +} + +func (c chaosAsyncCmd) CmdID() dispatcher.CommandID { + if c.mode == asyncUnknown { + return cmdChaosUnregistered + } + return cmdChaosAsync +} + +// chaosToken identifies the yield a completion answers. A completion that +// reaches an incarnation other than the one that issued it is a defect. +type chaosToken struct { + rec int + inc int + tag uint64 +} + +type chaosOpKind uint8 + +const ( + opSpin chaosOpKind = iota // n steps, each reporting Preempt or Continue + opAsync // yield and wait for the completion + opYieldPreempt // yield and report Preempt in the same step, then wait + opWaitMsg // wait until n messages arrived in total + opStaleUpgrade // yield with a long delay, preempt, upgrade without waiting + opUpgrade + opFail + opFinish + opNever // wait for termination +) + +type chaosOp struct { + delay time.Duration + n int + mode chaosAsyncMode + kind chaosOpKind +} + +// chaosRec is the shared state of one process across its incarnations. +type chaosRec struct { + result *apiruntime.Result + rng *rand.Rand + cancelParent context.CancelFunc + ready chan struct{} + pid pidapi.PID + plans [][]chaosOp + failUpgrade int + need int64 + id int + stepsRun atomic.Uint64 + msgs atomic.Int64 + maxSteps uint64 + resultMu sync.Mutex + killed atomic.Bool + upgrades atomic.Int32 + running atomic.Int32 + submitted atomic.Bool + gaveUp atomic.Bool + starts atomic.Int32 + completes atomic.Int32 + failStart bool + deaf bool + blocker bool + pooled bool + failUpInit bool +} + +func (r *chaosRec) expectsLimit() bool { return r.maxSteps > 0 } + +type chaosWorld struct { + sched *Scheduler + appCtx context.Context + t *testing.T + recs []*chaosRec + incs []*chaosProc + violations []string + seed int64 + mu sync.Mutex + stopped atomic.Bool +} + +func (w *chaosWorld) violatef(format string, args ...any) { + w.mu.Lock() + defer w.mu.Unlock() + if len(w.violations) < 40 { + w.violations = append(w.violations, fmt.Sprintf(format, args...)) + } +} + +func (w *chaosWorld) track(p *chaosProc) { + w.mu.Lock() + w.incs = append(w.incs, p) + w.mu.Unlock() +} + +// chaosProc is one incarnation of a chaos actor. +type chaosProc struct { + w *chaosWorld + rec *chaosRec + rng *rand.Rand + outstanding map[uint64]struct{} + plan []chaosOp + steps uint64 + nextTag uint64 + waitTag uint64 + pc int + ph int + inc int + closed atomic.Int32 + running atomic.Int32 + inits atomic.Int32 + preempt bool + limited bool +} + +var ( + _ process.Process = (*chaosProc)(nil) + _ process.Preemptible = (*chaosProc)(nil) + _ process.StepAccounted = (*chaosProc)(nil) +) + +func (p *chaosProc) EnablePreemption() { p.preempt = true } +func (p *chaosProc) StepsUsed() uint64 { return p.steps } +func (p *chaosProc) ResumeStepCount(used uint64) { p.steps = used } + +func (p *chaosProc) Init(context.Context, string, payload.Payloads) error { + if p.inits.Add(1) != 1 { + p.w.violatef("rec %d inc %d: Init called more than once", p.rec.id, p.inc) + } + if p.rec.failUpInit && p.inc > 0 { + return errChaosStart + } + return nil +} + +func (p *chaosProc) Close() { + if p.closed.Add(1) != 1 { + p.w.violatef("rec %d inc %d: closed more than once", p.rec.id, p.inc) + } + if p.running.Load() != 0 { + p.w.violatef("rec %d inc %d: closed during a step", p.rec.id, p.inc) + } +} + +func isCancelEvent(data any) bool { + pkg, ok := data.(*relay.Package) + if !ok || pkg == nil || len(pkg.Messages) != 1 || pkg.Messages[0].Topic != topology.TopicEvents || len(pkg.Messages[0].Payloads) != 1 { + return false + } + _, ok = pkg.Messages[0].Payloads[0].Data().(*topology.CancelEvent) + return ok +} + +func (p *chaosProc) Step(events []process.Event, out *process.StepOutput) error { + rec := p.rec + if !p.running.CompareAndSwap(0, 1) { + p.w.violatef("rec %d inc %d: concurrent steps of one incarnation", rec.id, p.inc) + } + defer p.running.Store(0) + if rec.running.Add(1) != 1 { + p.w.violatef("rec %d: steps of two incarnations overlap", rec.id) + } + defer rec.running.Add(-1) + + if rec.completes.Load() != 0 { + p.w.violatef("rec %d inc %d: step after completion", rec.id, p.inc) + } + if p.closed.Load() != 0 { + p.w.violatef("rec %d inc %d: step after close", rec.id, p.inc) + } + if p.limited { + p.w.violatef("rec %d inc %d: step after the step limit failed it", rec.id, p.inc) + } + + p.steps++ + if rec.maxSteps > 0 && p.steps > rec.maxSteps { + p.limited = true + return process.ErrStepLimitExceeded + } + total := rec.stepsRun.Add(1) + if rec.maxSteps > 0 && total > rec.maxSteps { + p.w.violatef("rec %d: %d steps ran with max_steps %d", rec.id, total, rec.maxSteps) + } + + cancelled := false + for _, e := range events { + switch e.Type { + case process.EventYieldComplete: + p.absorb(e) + case process.EventMessage: + if isCancelEvent(e.Data) { + cancelled = true + } else if e.Data != nil { + rec.msgs.Add(1) + } + } + } + if cancelled && !rec.deaf { + return errChaosCancel + } + return p.advance(out) +} + +func (p *chaosProc) absorb(e process.Event) { + rec := p.rec + if _, ok := p.outstanding[e.Tag]; !ok { + p.w.violatef("rec %d inc %d: completion for tag %d that is not outstanding (data %#v)", rec.id, p.inc, e.Tag, e.Data) + return + } + delete(p.outstanding, e.Tag) + if e.Error != nil { + return + } + tok, ok := e.Data.(chaosToken) + if !ok || tok.rec != rec.id || tok.inc != p.inc || tok.tag != e.Tag { + p.w.violatef("rec %d inc %d: completion %#v delivered to the wrong yield (tag %d)", rec.id, p.inc, e.Data, e.Tag) + } +} + +func (p *chaosProc) yield(out *process.StepOutput, op chaosOp) uint64 { + p.nextTag++ + tag := p.nextTag + p.outstanding[tag] = struct{}{} + out.Yield(chaosAsyncCmd{token: chaosToken{rec: p.rec.id, inc: p.inc, tag: tag}, delay: op.delay, mode: op.mode}, tag) + return tag +} + +func (p *chaosProc) next() { p.pc++; p.ph = 0 } + +func (p *chaosProc) advance(out *process.StepOutput) error { + for { + if p.pc >= len(p.plan) { + out.Done(payload.New(chaosToken{rec: p.rec.id, inc: p.inc})) + return nil + } + op := p.plan[p.pc] + switch op.kind { + case opSpin: + if p.ph >= op.n { + p.next() + continue + } + p.ph++ + if p.rng.Intn(3) == 0 { + out.Continue() + } else { + out.Preempt() + } + return nil + case opAsync: + if p.ph == 0 { + p.waitTag = p.yield(out, op) + p.ph = 1 + out.WaitForYields() + return nil + } + if _, pending := p.outstanding[p.waitTag]; pending { + out.WaitForYields() + return nil + } + p.next() + case opYieldPreempt: + if p.ph == 0 { + p.waitTag = p.yield(out, op) + p.ph = 1 + out.Preempt() + return nil + } + if _, pending := p.outstanding[p.waitTag]; pending { + out.WaitForYields() + return nil + } + p.next() + case opWaitMsg: + if p.rec.msgs.Load() >= int64(op.n) { + p.next() + continue + } + out.Idle() + return nil + case opStaleUpgrade: + if p.ph == 0 { + p.yield(out, op) + p.ph = 1 + out.Preempt() + return nil + } + out.SetUpgrade(&process.UpgradeRequest{Source: registry.NewID("chaos", strconv.Itoa(p.rec.id))}) + return nil + case opUpgrade: + out.SetUpgrade(&process.UpgradeRequest{Source: registry.NewID("chaos", strconv.Itoa(p.rec.id))}) + return nil + case opFail: + return errChaosFail + case opFinish: + out.Done(payload.New(chaosToken{rec: p.rec.id, inc: p.inc})) + return nil + case opNever: + out.Idle() + return nil + } + } +} + +type chaosFactory struct{ w *chaosWorld } + +func (f chaosFactory) Create(id registry.ID) (process.Process, *process.Meta, error) { + n, err := strconv.Atoi(id.Name) + if err != nil || n < 0 || n >= len(f.w.recs) { + return nil, nil, fmt.Errorf("unknown chaos source %v", id) + } + rec := f.w.recs[n] + inc := int(rec.upgrades.Add(1)) + if rec.failUpgrade == inc { + return nil, nil, errChaosStart + } + if inc >= len(rec.plans) { + f.w.violatef("rec %d: upgrade to incarnation %d but the plan has %d", rec.id, inc, len(rec.plans)) + return nil, nil, errChaosStart + } + return f.w.newProc(rec, inc), &process.Meta{Method: "main"}, nil +} + +func (w *chaosWorld) newProc(rec *chaosRec, inc int) *chaosProc { + p := &chaosProc{ + w: w, + rec: rec, + inc: inc, + plan: rec.plans[inc], + rng: rand.New(rand.NewSource(int64(rec.id)*7919 + int64(inc) + w.seed)), + outstanding: make(map[uint64]struct{}, 4), + } + w.track(p) + return p +} + +func chaosHandler() dispatcher.Handler { + return dispatcher.HandlerFunc(func(_ context.Context, cmd dispatcher.Command, tag uint64, r dispatcher.ResultReceiver) error { + c := cmd.(chaosAsyncCmd) + switch c.mode { + case asyncInline: + r.CompleteYield(tag, c.token, nil) + case asyncHandleErr: + return errChaosHandler + case asyncLaterError: + time.AfterFunc(c.delay, func() { r.CompleteYield(tag, nil, errChaosHandler) }) + default: + time.AfterFunc(c.delay, func() { r.CompleteYield(tag, c.token, nil) }) + } + return nil + }) +} + +type chaosLifecycle struct{ w *chaosWorld } + +func (l chaosLifecycle) recFor(p pidapi.PID) *chaosRec { + n, err := strconv.Atoi(strings.TrimPrefix(p.UniqID, "chaos-")) + if err != nil || n < 0 || n >= len(l.w.recs) { + return nil + } + return l.w.recs[n] +} + +func (l chaosLifecycle) OnStart(_ context.Context, p pidapi.PID, _ process.Process) error { + rec := l.recFor(p) + if rec == nil { + return nil + } + if rec.starts.Add(1) != 1 { + l.w.violatef("rec %d: started more than once", rec.id) + } + if rec.failStart { + return errChaosStart + } + return nil +} + +func (l chaosLifecycle) OnComplete(_ context.Context, p pidapi.PID, res *apiruntime.Result) { + rec := l.recFor(p) + if rec == nil { + return + } + if rec.running.Load() != 0 { + l.w.violatef("rec %d: completed during a step", rec.id) + } + if rec.completes.Add(1) != 1 { + l.w.violatef("rec %d: completed more than once", rec.id) + return + } + rec.resultMu.Lock() + rec.result = res + rec.resultMu.Unlock() +} + +func chaosPlanFor(rng *rand.Rand, rec *chaosRec) { + randDelay := func() time.Duration { + switch rng.Intn(10) { + case 0: + return 0 + case 1: + return time.Duration(5+rng.Intn(15)) * time.Millisecond + default: + return time.Duration(rng.Intn(1500)) * time.Microsecond + } + } + middle := func() []chaosOp { + var ops []chaosOp + for i := rng.Intn(5); i > 0; i-- { + switch rng.Intn(6) { + case 0: + ops = append(ops, chaosOp{kind: opSpin, n: 1 + rng.Intn(12)}) + case 1, 2: + ops = append(ops, chaosOp{kind: opAsync, delay: randDelay(), mode: chaosAsyncMode(rng.Intn(5))}) + case 3: + ops = append(ops, chaosOp{kind: opYieldPreempt, delay: randDelay(), mode: asyncLater}) + case 4: + ops = append(ops, chaosOp{kind: opAsync, delay: randDelay(), mode: asyncLaterError}) + default: + ops = append(ops, chaosOp{kind: opSpin, n: 1 + rng.Intn(3)}) + } + } + return ops + } + + kind := rng.Intn(10) + incarnations := 1 + if kind <= 3 { + incarnations = 2 + rng.Intn(3) + } + stale := false + for i := 0; i < incarnations; i++ { + plan := middle() + if stale { + // The replacement waits on a yield with the same tag the replaced + // code left outstanding. + plan = append([]chaosOp{{kind: opAsync, delay: 12 * time.Millisecond, mode: asyncLater}}, plan...) + stale = false + } + if i < incarnations-1 { + if rng.Intn(3) == 0 { + plan = append(plan, chaosOp{kind: opStaleUpgrade, delay: time.Duration(1+rng.Intn(4)) * time.Millisecond}) + stale = true + } else { + plan = append(plan, chaosOp{kind: opUpgrade}) + } + } else { + switch kind { + case 4: + plan = append(plan, chaosOp{kind: opFail}) + case 5: + rec.blocker = true + plan = append(plan, chaosOp{kind: opNever}) + case 6: + rec.need = int64(50 + rng.Intn(400)) + plan = append(plan, chaosOp{kind: opWaitMsg, n: int(rec.need)}, chaosOp{kind: opFinish}) + default: + plan = append(plan, chaosOp{kind: opFinish}) + } + } + if i == 0 && kind != 6 && rng.Intn(3) == 0 { + n := 1 + rng.Intn(20) + plan = append([]chaosOp{{kind: opWaitMsg, n: n}}, plan...) + if int64(n) > rec.need { + rec.need = int64(n) + } + } + rec.plans = append(rec.plans, plan) + } + if rng.Intn(5) == 0 { + rec.maxSteps = uint64(3 + rng.Intn(60)) + } + rec.deaf = rng.Intn(8) == 0 + if incarnations > 1 { + switch rng.Intn(10) { + case 0: + rec.failUpgrade = 1 + rng.Intn(incarnations-1) + case 1: + rec.failUpInit = true + } + } + rec.failStart = rng.Intn(25) == 0 +} + +func (w *chaosWorld) submit(rec *chaosRec) { + proc := w.newProc(rec, 0) + frame, fc := ctxapi.OpenFrameContext(w.appCtx) + fc.Seal() + ctx, cancel := context.WithCancel(frame) + rec.cancelParent = cancel + _, err := w.sched.Submit(ctx, rec.pid, proc, "main", nil) + if err != nil { + cancel() + proc.Close() + switch { + case errors.Is(err, process.ErrSchedulerStopping), errors.Is(err, process.ErrMaxProcessesExceeded), errors.Is(err, errChaosStart): + default: + w.violatef("rec %d: unexpected Submit error %v", rec.id, err) + } + if rec.completes.Load() != 0 { + w.violatef("rec %d: completed although Submit failed", rec.id) + } + rec.gaveUp.Store(true) + close(rec.ready) + return + } + rec.submitted.Store(true) + close(rec.ready) +} + +// submitPooled runs a process the way a pooled executor does: the owner +// creates the processor, schedules it, reads its result and releases it. +func (w *chaosWorld) submitPooled(rec *chaosRec) { + proc := w.newProc(rec, 0) + frame, fc := ctxapi.OpenFrameContext(w.appCtx) + fc.Seal() + ctx, cancel := context.WithCancel(frame) + defer cancel() + pr, err := w.sched.CreateProcessor(ctx, rec.pid, proc) + if err != nil { + proc.Close() + if !errors.Is(err, process.ErrSchedulerStopping) && !errors.Is(err, process.ErrMaxProcessesExceeded) { + w.violatef("rec %d: unexpected CreateProcessor error %v", rec.id, err) + } + rec.gaveUp.Store(true) + close(rec.ready) + return + } + rec.submitted.Store(true) + rec.cancelParent = cancel + close(rec.ready) + results := pr.resultCh + if err := proc.Init(ctx, "main", nil); err != nil { + w.sched.ReleaseProcessor(pr) + return + } + w.sched.global.Push(pr) + w.sched.wakeAny() + select { + case <-results: + case <-time.After(30 * time.Second): + w.violatef("rec %d: pooled execution did not complete", rec.id) + } + w.sched.ReleaseProcessor(pr) +} + +func (w *chaosWorld) allowedSendError(err error) bool { + return err == nil || errors.Is(err, process.ErrProcessNotFound) || errors.Is(err, process.ErrProcessClosed) +} + +func (w *chaosWorld) send(rec *chaosRec) { + err := w.sched.Send(&relay.Package{Target: rec.pid, Messages: []*relay.Message{{Topic: "chaos"}}}) + if !w.allowedSendError(err) { + w.violatef("rec %d: unexpected Send error %v", rec.id, err) + } +} + +func chaosGoroutineSettle(baseline int, d time.Duration) (int, bool) { + deadline := time.Now().Add(d) + for { + n := goruntime.NumGoroutine() + if n <= baseline+2 { + return n, true + } + if time.Now().After(deadline) { + return n, false + } + time.Sleep(10 * time.Millisecond) + } +} + +func TestSchedulerChaos(t *testing.T) { + seed := int64(chaosEnvInt("WIPPY_CHAOS_SEED", 1, 1)) + runs := chaosEnvInt("WIPPY_CHAOS_RUNS", 4, 200) + for i := 0; i < runs; i++ { + s := seed + int64(i) + t.Run(fmt.Sprintf("seed=%d", s), func(t *testing.T) { runSchedulerChaos(t, s) }) + } +} + +func runSchedulerChaos(t *testing.T, seed int64) { + procs := chaosEnvInt("WIPPY_CHAOS_PROCS", 120, 600) + rng := rand.New(rand.NewSource(seed)) + workers := 1 + rng.Intn(6) + maxProcs := int64(0) + if rng.Intn(4) == 0 { + maxProcs = int64(procs / 2) + } + stopMode := rng.Intn(3) // 0: stop after the run, 1: stop at a random time, 2: stop at a random time with a short deadline + t.Logf("chaos seed=%d workers=%d procs=%d maxProcs=%d stopMode=%d (replay: WIPPY_CHAOS_SEED=%d WIPPY_CHAOS_RUNS=1)", seed, workers, procs, maxProcs, stopMode, seed) + + baseline := goruntime.NumGoroutine() + + w := &chaosWorld{t: t, seed: seed} + reg := scheduler.NewRegistry() + reg.Register(cmdChaosAsync, chaosHandler()) + opts := []Option{WithWorkers(workers), WithLifecycle(chaosLifecycle{w: w}), WithLocalQueueSize(8 << rng.Intn(5))} + if maxProcs > 0 { + opts = append(opts, WithMaxProcesses(maxProcs)) + } + w.sched = NewScheduler(reg, opts...) + + appCtx := ctxapi.WithAppContext(context.Background(), ctxapi.NewAppContext()) + process.WithFactory(appCtx, chaosFactory{w: w}) + w.appCtx = appCtx + + for i := 0; i < procs; i++ { + rec := &chaosRec{id: i, pid: pidapi.PID{UniqID: fmt.Sprintf("chaos-%d", i)}, ready: make(chan struct{})} + rec.rng = rand.New(rand.NewSource(seed*1000003 + int64(i))) + chaosPlanFor(rec.rng, rec) + rec.pooled = rec.rng.Intn(12) == 0 && len(rec.plans) == 1 + if rec.pooled { + rec.failStart = false + rec.failUpgrade = 0 + } + w.recs = append(w.recs, rec) + } + + w.sched.Start() + + var wg sync.WaitGroup + for _, rec := range w.recs { + wg.Add(1) + go func() { + defer wg.Done() + time.Sleep(time.Duration(rec.rng.Intn(40_000)) * time.Microsecond) + if rec.pooled { + w.submitPooled(rec) + } else { + w.submit(rec) + } + }() + wg.Add(1) + go func() { + defer wg.Done() + <-rec.ready + if !rec.submitted.Load() { + return + } + r := rec.rng + for sent := int64(0); sent < rec.need+int64(r.Intn(8)); sent++ { + w.send(rec) + if r.Intn(16) == 0 { + time.Sleep(time.Duration(r.Intn(500)) * time.Microsecond) + } + } + w.send(&chaosRec{pid: pidapi.PID{UniqID: "chaos-missing"}, id: -1}) + time.Sleep(time.Duration(r.Intn(15)) * time.Millisecond) + if rec.blocker || r.Intn(7) == 0 { + rec.killed.Store(true) + if r.Intn(2) == 0 { + rec.cancelParent() + } else if err := w.sched.Terminate(rec.pid); err != nil && !errors.Is(err, process.ErrProcessNotFound) { + w.violatef("rec %d: unexpected Terminate error %v", rec.id, err) + } + } + }() + } + + if rng.Intn(2) == 0 { + resizeSeed := rng.Int63() + wg.Add(1) + go func() { + defer wg.Done() + r := rand.New(rand.NewSource(resizeSeed)) + for i := 0; i < 20; i++ { + time.Sleep(time.Duration(r.Intn(4000)) * time.Microsecond) + if err := w.sched.ResizeWorkers(1 + r.Intn(6)); err != nil && !errors.Is(err, process.ErrSchedulerStopping) { + w.violatef("unexpected ResizeWorkers error %v", err) + } + } + }() + } + + stopTimeout := 20 * time.Second + if stopMode == 2 { + stopTimeout = time.Duration(20+rng.Intn(80)) * time.Millisecond + } + stopDone := make(chan struct{}) + startStop := func() { + w.stopped.Store(true) + started := time.Now() + ctx, cancel := context.WithTimeout(context.Background(), stopTimeout) + w.sched.Stop(ctx) + cancel() + if stopMode != 2 && time.Since(started) > stopTimeout-time.Second { + w.violatef("Stop took %v, close to its %v deadline", time.Since(started), stopTimeout) + } + close(stopDone) + } + if stopMode != 0 { + time.AfterFunc(time.Duration(rng.Intn(60))*time.Millisecond, func() { go startStop() }) + } + + wg.Wait() + if stopMode == 0 { + w.awaitSettled(t, 30*time.Second) + go startStop() + } + select { + case <-stopDone: + case <-time.After(60 * time.Second): + t.Fatalf("seed %d: Stop did not return (goroutines dumped below)\n%s", seed, chaosStacks()) + } + w.awaitSettled(t, 10*time.Second) + + w.verify(t, seed) + + if n, ok := chaosGoroutineSettle(baseline, 10*time.Second); !ok { + t.Errorf("seed %d: %d goroutines remain, baseline %d\n%s", seed, n, baseline, chaosStacks()) + } +} + +func chaosStacks() string { + buf := make([]byte, 1<<20) + return string(buf[:goruntime.Stack(buf, true)]) +} + +// awaitSettled waits until every record reached a terminal state. +func (w *chaosWorld) awaitSettled(t *testing.T, d time.Duration) { + t.Helper() + deadline := time.Now().Add(d) + for { + var stranded []string + for _, rec := range w.recs { + select { + case <-rec.ready: + default: + stranded = append(stranded, fmt.Sprintf("rec %d never submitted", rec.id)) + continue + } + if rec.submitted.Load() && rec.completes.Load() == 0 { + stranded = append(stranded, fmt.Sprintf("rec %d (pooled=%v blocker=%v deaf=%v plans=%d) stranded", rec.id, rec.pooled, rec.blocker, rec.deaf, len(rec.plans))) + } + } + if len(stranded) == 0 { + return + } + if time.Now().After(deadline) { + t.Errorf("seed %d: %d processes stranded: %v\nprocesses: %+v", w.seed, len(stranded), stranded[:min(len(stranded), 10)], w.sched.ListProcesses()) + return + } + time.Sleep(5 * time.Millisecond) + } +} + +func (w *chaosWorld) verify(t *testing.T, seed int64) { + t.Helper() + outcomes := map[string]int{} + upgrades, rejected := 0, 0 + for _, rec := range w.recs { + starts, completes := rec.starts.Load(), rec.completes.Load() + if rec.gaveUp.Load() { + rejected++ + if completes != 0 { + w.violatef("rec %d: completed although admission failed", rec.id) + } + continue + } + if completes != 1 { + w.violatef("rec %d: completed %d times (starts %d)", rec.id, completes, starts) + continue + } + rec.resultMu.Lock() + res := rec.result + rec.resultMu.Unlock() + w.checkOutcome(rec, res) + outcomes[outcomeName(res)]++ + upgrades += int(rec.upgrades.Load()) + } + t.Logf("seed %d outcomes %v admitted-failures=%d upgrades=%d", seed, outcomes, rejected, upgrades) + + w.mu.Lock() + incs := append([]*chaosProc(nil), w.incs...) + w.mu.Unlock() + for _, p := range incs { + if p.inits.Load() > 0 && p.closed.Load() != 1 { + w.violatef("rec %d inc %d: closed %d times, want once", p.rec.id, p.inc, p.closed.Load()) + } + } + + s := w.sched + if n := s.parked.Load(); n != 0 { + w.violatef("parked counter = %d after stop", n) + } + for _, wk := range s.workerSnapshot() { + if wk.parked.Load() { + w.violatef("worker %d still counted as parked", wk.id) + } + } + if n := s.processorCount.Load(); n != 0 { + w.violatef("processor count = %d after stop", n) + } + s.admitMu.Lock() + if s.admitting != 0 { + w.violatef("admissions in flight = %d after stop", s.admitting) + } + s.admitMu.Unlock() + n := 0 + s.byPID.Range(func(_, _ any) bool { n++; return true }) + if n != 0 { + w.violatef("%d processors still registered by pid", n) + } + n = 0 + s.byQueue.Range(func(_, _ any) bool { n++; return true }) + if n != 0 { + w.violatef("%d processors still registered by queue", n) + } + if l := s.global.Len(); l > 0 { + w.violatef("global queue holds %d entries after stop", l) + } + + if len(w.violations) > 0 { + t.Errorf("seed %d: %d invariant violations (replay: WIPPY_CHAOS_SEED=%d WIPPY_CHAOS_RUNS=1):\n %s", + seed, len(w.violations), seed, strings.Join(w.violations, "\n ")) + } +} + +// checkOutcome verifies the process ended the way its plan and the external +// events that hit it allow. +func (w *chaosWorld) checkOutcome(rec *chaosRec, res *apiruntime.Result) { + if res == nil { + w.violatef("rec %d: no result", rec.id) + return + } + err := res.Error + killed := rec.killed.Load() || w.stopped.Load() + switch { + case err == nil: + var tok chaosToken + var ok bool + if res.Value != nil { + tok, ok = res.Value.Data().(chaosToken) + } + if !ok || tok.rec != rec.id || tok.inc != len(rec.plans)-1 { + w.violatef("rec %d: result %v does not come from the final incarnation %d", rec.id, res.Value, len(rec.plans)-1) + } + last := rec.plans[len(rec.plans)-1] + if k := last[len(last)-1].kind; k == opFail || k == opNever { + w.violatef("rec %d: finished although its plan ends with kind %d", rec.id, k) + } + case errors.Is(err, process.ErrStepLimitExceeded): + if !rec.expectsLimit() { + w.violatef("rec %d: step limit error without max_steps", rec.id) + } + case errors.Is(err, errChaosFail): + last := rec.plans[len(rec.plans)-1] + if last[len(last)-1].kind != opFail { + w.violatef("rec %d: planned failure but plan does not fail", rec.id) + } + case errors.Is(err, sysprocess.ErrTerminated), errors.Is(err, errChaosCancel), errors.Is(err, context.Canceled): + if !killed { + w.violatef("rec %d: terminated without a terminate or stop (%v)", rec.id, err) + } + case errors.Is(err, errChaosStart): + if rec.failUpgrade == 0 && !rec.failUpInit { + w.violatef("rec %d: upgrade failure without injected failure (%v)", rec.id, err) + } + default: + w.violatef("rec %d: unexpected error %v", rec.id, err) + } + if err != nil && !killed && !rec.expectsLimit() && rec.failUpgrade == 0 && !rec.failUpInit { + last := rec.plans[len(rec.plans)-1] + if last[len(last)-1].kind != opFail { + w.violatef("rec %d: unprovoked failure %v", rec.id, err) + } + } + if err == nil && rec.blocker && !killed { + w.violatef("rec %d: blocker finished", rec.id) + } +} + +func outcomeName(res *apiruntime.Result) string { + switch err := res.Error; { + case err == nil: + return "done" + case errors.Is(err, process.ErrStepLimitExceeded): + return "limit" + case errors.Is(err, errChaosFail): + return "fail" + case errors.Is(err, errChaosStart): + return "upgrade-fail" + default: + return "killed" + } +} diff --git a/system/scheduler/actor/close_ownership_test.go b/system/scheduler/actor/close_ownership_test.go new file mode 100644 index 000000000..20956d85e --- /dev/null +++ b/system/scheduler/actor/close_ownership_test.go @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: MPL-2.0 + +package actor + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + ctxapi "github.com/wippyai/runtime/api/context" + "github.com/wippyai/runtime/api/dispatcher" + "github.com/wippyai/runtime/api/payload" + pidapi "github.com/wippyai/runtime/api/pid" + "github.com/wippyai/runtime/api/process" + "github.com/wippyai/runtime/api/registry" + "github.com/wippyai/runtime/api/relay" + apiruntime "github.com/wippyai/runtime/api/runtime" + "github.com/wippyai/runtime/system/scheduler" +) + +// closeCountingProcess counts Close calls. +type closeCountingProcess struct { + upgradeSource + closed atomic.Int64 +} + +func (p *closeCountingProcess) Close() { p.closed.Add(1) } + +// A process whose admission fails stays owned by the submitter, which closes it. +func TestSubmitRejectedByLifecycleDoesNotCloseProcess(t *testing.T) { + sched := newPreemptTestScheduler(1, &testLifecycle{}) + sched.lifecycle = &rejectingLifecycle2{} + sched.Start() + defer testStopScheduler(sched) + + p := &closeCountingProcess{} + if _, err := sched.Submit(context.Background(), pidapi.PID{UniqID: "r"}, p, "", nil); err == nil { + t.Fatal("expected rejection") + } + if got := p.closed.Load(); got != 0 { + t.Fatalf("scheduler closed a process it did not admit (%d)", got) + } +} + +// Every failed upgrade closes the process exactly once. +func TestFailedUpgradeClosesProcessOnce(t *testing.T) { + cases := map[string]func(*closeCountingProcess) (process.Factory, *upgradeFailure){ + "create": func(*closeCountingProcess) (process.Factory, *upgradeFailure) { + return &mockFactory{createFunc: func(registry.ID) (process.Process, *process.Meta, error) { + return nil, nil, errors.New("create failed") + }}, nil + }, + "init": func(*closeCountingProcess) (process.Factory, *upgradeFailure) { + target := &upgradeFailure{} + return &mockFactory{createFunc: func(registry.ID) (process.Process, *process.Meta, error) { + return target, &process.Meta{Method: "main"}, nil + }}, target + }, + } + for name, setup := range cases { + t.Run(name, func(t *testing.T) { + done := make(chan struct{}) + lc := &testLifecycle{onComplete: func(context.Context, pidapi.PID, *apiruntime.Result) { close(done) }} + sched := newPreemptTestScheduler(1, lc) + sched.Start() + defer testStopScheduler(sched) + + source := &closeCountingProcess{upgradeSource: upgradeSource{UpgradeProcess: UpgradeProcess{ + upgradeReq: &process.UpgradeRequest{Source: registry.NewID("app", "next")}}}} + factory, target := setup(source) + appCtx := ctxapi.WithAppContext(context.Background(), ctxapi.NewAppContext()) + process.WithFactory(appCtx, factory) + frameCtx, fc := ctxapi.OpenFrameContext(appCtx) + fc.Seal() + if _, err := sched.Submit(frameCtx, pidapi.PID{UniqID: "u"}, source, "", nil); err != nil { + t.Fatal(err) + } + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("actor did not complete") + } + // Stopping joins the workers, so every Close has happened. + testStopScheduler(sched) + closes := source.closed.Load() + if target != nil { + closes += target.closed.Load() + } + wantTotal := int64(1) + if target != nil { + wantTotal = 2 + } + if closes != wantTotal { + t.Fatalf("expected %d closes across incarnations, got %d", wantTotal, closes) + } + if target != nil && target.closed.Load() != 1 { + t.Fatalf("replacement closed %d times", target.closed.Load()) + } + }) + } +} + +// upgradeFailure is a replacement whose Init fails. +type upgradeFailure struct { + upgradeTarget + closed atomic.Int64 +} + +func (p *upgradeFailure) Init(context.Context, string, payload.Payloads) error { + return errors.New("init failed") +} +func (p *upgradeFailure) Close() { p.closed.Add(1) } + +// waitingProcess yields once and then waits for the completion. +type waitingProcess struct { + steps atomic.Int64 +} + +func (*waitingProcess) Init(context.Context, string, payload.Payloads) error { return nil } +func (p *waitingProcess) Step(events []process.Event, out *process.StepOutput) error { + if p.steps.Add(1) == 1 { + out.Yield(YieldCmd{}, 7) + out.WaitForYields() + return nil + } + if len(events) > 0 { + out.Done(nil) + return nil + } + out.WaitForYields() + return nil +} +func (*waitingProcess) Send(*relay.Package) error { return nil } +func (*waitingProcess) Close() {} + +// A completion that arrives after its actor ended is dropped, even when the +// processor slot has been reused by another actor. +// Yield handlers complete through the incarnation's generation-bound +// completer, never the pooled Processor: a completion that outlives the +// incarnation is dropped by the queue (TestA03StaleCompletionDoesNotEnqueue) +// instead of reaching the processor's next user. +func TestStaleYieldCompletionDoesNotWakeReusedProcessor(t *testing.T) { + receivers := make(chan dispatcher.ResultReceiver, 2) + reg := scheduler.NewRegistry() + reg.Register(CmdYield, dispatcher.HandlerFunc(func(_ context.Context, _ dispatcher.Command, _ uint64, r dispatcher.ResultReceiver) error { + receivers <- r + return nil + })) + reg.Register(CmdComplete, CompleteHandler()) + completions := make(chan string, 2) + lc := &testLifecycle{onComplete: func(_ context.Context, id pidapi.PID, _ *apiruntime.Result) { completions <- id.UniqID }} + sched := NewScheduler(reg, WithWorkers(1), WithLifecycle(lc)) + sched.Start() + defer testStopScheduler(sched) + + ctx, cancel := context.WithCancel(context.Background()) + if _, err := sched.Submit(ctx, pidapi.PID{UniqID: "a"}, &waitingProcess{}, "", nil); err != nil { + t.Fatal(err) + } + stale, ok := receive(t, receivers, "first yield").(*process.YieldCompleter) + if !ok { + t.Fatal("handlers complete through the incarnation's completer") + } + cancel() + if got := <-completions; got != "a" { + t.Fatalf("completed %q, want a", got) + } + + // The next process, which may reuse the finished one's processor slot, + // waits on its own tag 7; the finished incarnation's completer is inert. + next := &waitingProcess{} + if _, err := sched.Submit(context.Background(), pidapi.PID{UniqID: "b"}, next, "", nil); err != nil { + t.Fatal(err) + } + fresh := receive(t, receivers, "second yield") + stale.CompleteYield(7, nil, nil) + fresh.CompleteYield(7, nil, nil) + if got := <-completions; got != "b" { + t.Fatalf("completed %q, want b", got) + } + if got := next.steps.Load(); got != 2 { + t.Fatalf("the next process took %d steps, want 2: the stale completion cost it a step", got) + } +} diff --git a/system/scheduler/actor/edge_cases_test.go b/system/scheduler/actor/edge_cases_test.go index af343f681..b1644acfd 100644 --- a/system/scheduler/actor/edge_cases_test.go +++ b/system/scheduler/actor/edge_cases_test.go @@ -867,11 +867,9 @@ func TestCollectProcessStatsWithProvider(t *testing.T) { func TestWakeProcessorPath(t *testing.T) { reg := scheduler.NewRegistry() reg.Register(CmdYield, dispatcher.HandlerFunc(func(_ context.Context, _ dispatcher.Command, tag uint64, receiver dispatcher.ResultReceiver) error { - proc := receiver.(*Processor) - completer := proc.queue.NewYieldCompleter(proc.scheduler) go func() { time.Sleep(10 * time.Millisecond) - completer.CompleteYield(tag, nil, nil) + receiver.CompleteYield(tag, nil, nil) }() return nil })) diff --git a/system/scheduler/actor/inject.go b/system/scheduler/actor/inject.go index 1fe537dd3..d44f07582 100644 --- a/system/scheduler/actor/inject.go +++ b/system/scheduler/actor/inject.go @@ -70,6 +70,12 @@ func (q *InjectQueue) Pop() *Processor { return proc } +// IsEmpty reports whether no processor is queued. Only safe to call from the +// single consumer. +func (q *InjectQueue) IsEmpty() bool { + return q.tail.next.Load() == nil +} + // Drain pops all available processors into dst, returns count. // Useful for batch processing. func (q *InjectQueue) Drain(dst []*Processor) int { diff --git a/system/scheduler/actor/inject_test.go b/system/scheduler/actor/inject_test.go new file mode 100644 index 000000000..9e7da6e29 --- /dev/null +++ b/system/scheduler/actor/inject_test.go @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: MPL-2.0 + +package actor + +import "testing" + +func TestInjectQueueIsEmpty(t *testing.T) { + q := NewInjectQueue() + if !q.IsEmpty() { + t.Fatal("a new queue is empty") + } + q.Push(&Processor{}) + if q.IsEmpty() { + t.Fatal("a queue holding a processor is not empty") + } + if q.Pop() == nil { + t.Fatal("expected the queued processor") + } + if !q.IsEmpty() { + t.Fatal("a drained queue is empty") + } +} diff --git a/system/scheduler/actor/pooled_lifecycle_test.go b/system/scheduler/actor/pooled_lifecycle_test.go new file mode 100644 index 000000000..c33e1e3e9 --- /dev/null +++ b/system/scheduler/actor/pooled_lifecycle_test.go @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: MPL-2.0 + +package actor + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/wippyai/runtime/api/payload" + pidapi "github.com/wippyai/runtime/api/pid" + "github.com/wippyai/runtime/api/process" + "github.com/wippyai/runtime/api/relay" + sysprocess "github.com/wippyai/runtime/system/process" +) + +// idleForeverProcess waits for messages and never finishes on its own. +type idleForeverProcess struct{} + +func (idleForeverProcess) Init(context.Context, string, payload.Payloads) error { return nil } +func (idleForeverProcess) Step(_ []process.Event, out *process.StepOutput) error { + out.Idle() + return nil +} +func (idleForeverProcess) Send(*relay.Package) error { return nil } +func (idleForeverProcess) Close() {} + +func waitProcessorState(t *testing.T, proc *Processor, want ProcessState) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for ProcessState(proc.state.Load())&stateMask != want { + if time.Now().After(deadline) { + t.Fatalf("processor did not reach %s, is %s", StateName(want), StateName(ProcessState(proc.state.Load()))) + } + time.Sleep(time.Millisecond) + } +} + +// Terminate ends a pooled execution that waits for messages: the owner +// scheduled it, so its result must arrive. +func TestTerminateEndsIdlePooledProcessor(t *testing.T) { + sched := newTestSchedulerWithLifecycle(2, &testLifecycle{}) + sched.Start() + defer testStopScheduler(sched) + + id := pidapi.PID{UniqID: "pooled-idle"} + proc, err := sched.CreateProcessor(context.Background(), id, idleForeverProcess{}) + if err != nil { + t.Fatal(err) + } + results := proc.resultCh + sched.global.Push(proc) + sched.wakeAny() + waitProcessorState(t, proc, StateIdle) + + if err := sched.Terminate(id); err != nil { + t.Fatal(err) + } + select { + case res := <-results: + if !errors.Is(res.Error, sysprocess.ErrTerminated) { + t.Fatalf("expected termination, got %v", res.Error) + } + case <-time.After(5 * time.Second): + t.Fatal("terminated pooled execution never completed") + } + sched.ReleaseProcessor(proc) +} + +// Stop returns as soon as the last pooled processor is released. +func TestStopReturnsWhenLastPooledProcessorIsReleased(t *testing.T) { + sched := newTestSchedulerWithLifecycle(2, &testLifecycle{}) + sched.Start() + + proc, err := sched.CreateProcessor(context.Background(), pidapi.PID{UniqID: "pooled-held"}, idleForeverProcess{}) + if err != nil { + t.Fatal(err) + } + + stopped := make(chan struct{}) + go func() { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + sched.Stop(ctx) + close(stopped) + }() + deadline := time.Now().Add(5 * time.Second) + for !sched.isStopping() { + if time.Now().After(deadline) { + t.Fatal("stop did not begin") + } + time.Sleep(time.Millisecond) + } + time.Sleep(50 * time.Millisecond) + sched.ReleaseProcessor(proc) + + select { + case <-stopped: + case <-time.After(5 * time.Second): + t.Fatal("Stop kept waiting after the last processor was released") + } +} diff --git a/system/scheduler/actor/preempt_bench_test.go b/system/scheduler/actor/preempt_bench_test.go new file mode 100644 index 000000000..d52ca3949 --- /dev/null +++ b/system/scheduler/actor/preempt_bench_test.go @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: MPL-2.0 + +package actor + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "testing" + + "github.com/wippyai/runtime/api/payload" + pidapi "github.com/wippyai/runtime/api/pid" + "github.com/wippyai/runtime/api/process" + "github.com/wippyai/runtime/api/relay" + "github.com/wippyai/runtime/api/runtime" +) + +var benchSink atomic.Uint64 + +// sliceProcess burns a fixed amount of work per step and reports preemption +// until the shared slice budget is used up. +type sliceProcess struct { + slices *atomic.Int64 + work int +} + +func (*sliceProcess) Init(context.Context, string, payload.Payloads) error { return nil } +func (p *sliceProcess) EnablePreemption() {} +func (p *sliceProcess) Step(_ []process.Event, out *process.StepOutput) error { + if p.slices.Add(-1) < 0 { + out.Done(nil) + return nil + } + var acc uint64 + for i := 0; i < p.work; i++ { + acc += uint64(i) * 2654435761 + } + benchSink.Add(acc & 1) + out.Preempt() + return nil +} +func (*sliceProcess) Send(*relay.Package) error { return nil } +func (*sliceProcess) Close() {} + +// BenchmarkPreemptedSlices measures scheduler cost per preempted step with +// one or two spinning actors per worker; ns/op is wall time per slice. +func BenchmarkPreemptedSlices(b *testing.B) { + for _, workers := range []int{1, 2, 4, 8, 16} { + for _, perWorker := range []int{1, 2} { + b.Run(fmt.Sprintf("workers=%d/actors=%dx", workers, perWorker), func(b *testing.B) { + const work = 500 + var wg sync.WaitGroup + lc := &testLifecycle{ + onComplete: func(context.Context, pidapi.PID, *runtime.Result) { wg.Done() }, + } + sched := newPreemptTestScheduler(workers, lc) + sched.Start() + defer testStopScheduler(sched) + + var slices atomic.Int64 + slices.Store(int64(b.N)) + actors := workers * perWorker + wg.Add(actors) + b.ResetTimer() + for i := 0; i < actors; i++ { + p := &sliceProcess{slices: &slices, work: work} + if _, err := sched.Submit(context.Background(), pidapi.PID{UniqID: fmt.Sprintf("spin-%d", i)}, p, "", nil); err != nil { + b.Fatal(err) + } + } + wg.Wait() + }) + } + } +} + +// BenchmarkPreemptedRequeue drives workers by hand, one goroutine per worker +// and without parking, to measure the dispatch and requeue cost of a +// preempted step; ns/op is per slice across all workers. +func BenchmarkPreemptedRequeue(b *testing.B) { + for _, workers := range []int{1, 4, 16} { + for _, perWorker := range []int{1, 2} { + b.Run(fmt.Sprintf("workers=%d/actors=%dx", workers, perWorker), func(b *testing.B) { + sched := newPreemptTestScheduler(workers, &testLifecycle{}) + var slices atomic.Int64 + slices.Store(int64(b.N) + int64(workers*perWorker)) + for i := 0; i < workers*perWorker; i++ { + p := &sliceProcess{slices: &slices} + if _, err := sched.Submit(context.Background(), pidapi.PID{UniqID: fmt.Sprintf("spin-%d", i)}, p, "", nil); err != nil { + b.Fatal(err) + } + } + snapshot := sched.workerSnapshot() + var next atomic.Int32 + b.ResetTimer() + var wg sync.WaitGroup + for g := 0; g < workers; g++ { + wg.Add(1) + go func() { + defer wg.Done() + w := snapshot[int(next.Add(1))-1] + for slices.Load() > 0 { + if proc := w.findWork(); proc != nil { + w.executeOne(proc) + } + } + }() + } + wg.Wait() + }) + } + } +} diff --git a/system/scheduler/actor/preempt_leak_test.go b/system/scheduler/actor/preempt_leak_test.go new file mode 100644 index 000000000..4eb87bcb9 --- /dev/null +++ b/system/scheduler/actor/preempt_leak_test.go @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: MPL-2.0 + +package actor + +import ( + "context" + "fmt" + "runtime" + "sync/atomic" + "testing" + "time" + + "github.com/wippyai/runtime/api/payload" + pidapi "github.com/wippyai/runtime/api/pid" + "github.com/wippyai/runtime/api/process" + "github.com/wippyai/runtime/api/relay" + apiruntime "github.com/wippyai/runtime/api/runtime" +) + +// preemptNProcess reports StepPreempted n times, then completes. +type preemptNProcess struct { + closed *atomic.Int64 + // started, when set, receives a value on the first step. + started chan<- struct{} + left int + began bool +} + +func (*preemptNProcess) Init(context.Context, string, payload.Payloads) error { return nil } +func (p *preemptNProcess) Step(_ []process.Event, out *process.StepOutput) error { + if !p.began { + p.began = true + if p.started != nil { + p.started <- struct{}{} + } + } + if p.left == 0 { + out.Done(nil) + return nil + } + p.left-- + out.Preempt() + return nil +} +func (*preemptNProcess) Send(*relay.Package) error { return nil } +func (p *preemptNProcess) Close() { p.closed.Add(1) } + +func settledGoroutines(t *testing.T, want int) int { + t.Helper() + var n int + for i := 0; i < 200; i++ { + runtime.GC() + n = runtime.NumGoroutine() + if n <= want { + return n + } + time.Sleep(10 * time.Millisecond) + } + return n +} + +func TestPreemptCyclesReleaseEveryProcessAndGoroutine(t *testing.T) { + baseline := settledGoroutines(t, 0) + + const total = 3000 + var closed atomic.Int64 + completed := make(chan struct{}, total) + lc := &testLifecycle{ + onComplete: func(context.Context, pidapi.PID, *apiruntime.Result) { completed <- struct{}{} }, + } + sched := newPreemptTestScheduler(4, lc) + sched.Start() + + for i := 0; i < total; i++ { + p := &preemptNProcess{closed: &closed, left: i % 7} + id := pidapi.PID{UniqID: fmt.Sprintf("p%d", i)} + if _, err := sched.Submit(context.Background(), id, p, "", nil); err != nil { + t.Fatal(err) + } + } + + timeout := time.After(20 * time.Second) + for i := 0; i < total; i++ { + select { + case <-completed: + case <-timeout: + t.Fatalf("completed %d of %d", i, total) + } + } + + // Idle workers are the only parked ones, each counted once. + deadline := time.Now().Add(5 * time.Second) + for sched.parked.Load() != 4 && time.Now().Before(deadline) { + time.Sleep(5 * time.Millisecond) + } + if got := sched.parked.Load(); got != 4 { + t.Fatalf("expected 4 parked workers while idle, got %d", got) + } + // A parked worker has returned from its last step, so every Close ran. + if got := closed.Load(); got != total { + t.Fatalf("closed %d of %d processes", got, total) + } + + testStopScheduler(sched) + if got := sched.parked.Load(); got != 0 { + t.Fatalf("parked counter drifted to %d after stop", got) + } + if got := settledGoroutines(t, baseline); got > baseline { + t.Fatalf("goroutines grew from %d to %d", baseline, got) + } +} + +// Processes stopped while still preempting are closed and unregistered. +func TestStopClosesPreemptingProcesses(t *testing.T) { + var closed atomic.Int64 + sched := newPreemptTestScheduler(2, &testLifecycle{}) + sched.Start() + const total = 20 + started := make(chan struct{}, total) + for i := 0; i < total; i++ { + p := &preemptNProcess{closed: &closed, left: 1 << 30, started: started} + if _, err := sched.Submit(context.Background(), pidapi.PID{UniqID: fmt.Sprintf("s%d", i)}, p, "", nil); err != nil { + t.Fatal(err) + } + } + for i := 0; i < total; i++ { + <-started + } + testStopScheduler(sched) + if got := closed.Load(); got != total { + t.Fatalf("closed %d of %d processes at stop", got, total) + } + if got := sched.parked.Load(); got != 0 { + t.Fatalf("parked counter is %d after stop", got) + } +} diff --git a/system/scheduler/actor/preempt_test.go b/system/scheduler/actor/preempt_test.go new file mode 100644 index 000000000..b8ee86243 --- /dev/null +++ b/system/scheduler/actor/preempt_test.go @@ -0,0 +1,371 @@ +// SPDX-License-Identifier: MPL-2.0 + +package actor + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/wippyai/runtime/api/attrs" + ctxapi "github.com/wippyai/runtime/api/context" + "github.com/wippyai/runtime/api/payload" + pidapi "github.com/wippyai/runtime/api/pid" + "github.com/wippyai/runtime/api/process" + "github.com/wippyai/runtime/api/registry" + "github.com/wippyai/runtime/api/relay" + "github.com/wippyai/runtime/api/runtime" + "github.com/wippyai/runtime/system/scheduler" +) + +// spinProcess never finishes on its own: every step uses up its budget. +type spinProcess struct { + enabled atomic.Bool + release atomic.Bool + steps atomic.Int64 +} + +func (*spinProcess) Init(context.Context, string, payload.Payloads) error { return nil } +func (p *spinProcess) EnablePreemption() { p.enabled.Store(true) } +func (p *spinProcess) Step(_ []process.Event, out *process.StepOutput) error { + p.steps.Add(1) + if p.release.Load() { + out.Done(nil) + return nil + } + out.Preempt() + return nil +} +func (*spinProcess) Send(*relay.Package) error { return nil } +func (*spinProcess) Close() {} + +// pingProcess yields rounds times and completes. +type pingProcess struct { + rounds int + done int +} + +func (*pingProcess) Init(context.Context, string, payload.Payloads) error { return nil } +func (p *pingProcess) Step(events []process.Event, out *process.StepOutput) error { + p.done += len(events) + if p.done >= p.rounds { + out.Done(nil) + return nil + } + if len(events) > 0 || p.done == 0 { + out.Yield(YieldCmd{}, uint64(p.done+1)) + } + out.WaitForYields() + return nil +} +func (*pingProcess) Send(*relay.Package) error { return nil } +func (*pingProcess) Close() {} + +// preemptWithPendingYieldProcess dispatches a long sleep and reports +// preemption in the same step; it must run again without the sleep finishing. +type preemptWithPendingYieldProcess struct { + steps atomic.Int64 +} + +func (*preemptWithPendingYieldProcess) Init(context.Context, string, payload.Payloads) error { + return nil +} +func (p *preemptWithPendingYieldProcess) Step(_ []process.Event, out *process.StepOutput) error { + if p.steps.Add(1) == 1 { + out.Yield(SleepCmd{Duration: time.Hour}, 1) + out.Preempt() + return nil + } + out.Done(nil) + return nil +} +func (*preemptWithPendingYieldProcess) Send(*relay.Package) error { return nil } +func (*preemptWithPendingYieldProcess) Close() {} + +// newPingLifecycle returns a lifecycle whose channel closes when the process +// with pid "ping" completes. +func newPingLifecycle() (*testLifecycle, <-chan struct{}) { + done := make(chan struct{}) + return &testLifecycle{ + onComplete: func(_ context.Context, pid pidapi.PID, _ *runtime.Result) { + if pid.UniqID == "ping" { + close(done) + } + }, + }, done +} + +func newPreemptTestScheduler(workers int, lc *testLifecycle, opts ...Option) *Scheduler { + registry := scheduler.NewRegistry() + registry.Register(CmdComplete, CompleteHandler()) + registry.Register(CmdYield, YieldHandler()) + registry.Register(CmdSleep, SleepHandler()) + return NewScheduler(registry, append([]Option{WithWorkers(workers), WithLifecycle(lc)}, opts...)...) +} + +func TestSchedulerEnablesPreemptionForPreemptibleProcesses(t *testing.T) { + sched := newPreemptTestScheduler(1, &testLifecycle{}) + sched.Start() + defer testStopScheduler(sched) + p := &spinProcess{} + p.release.Store(true) + if _, err := sched.Submit(context.Background(), pidapi.PID{UniqID: "a"}, p, "", nil); err != nil { + t.Fatal(err) + } + if !p.enabled.Load() { + t.Fatal("expected preemption to be enabled at submit") + } + q := &spinProcess{} + q.release.Store(true) + if _, err := sched.CreateProcessor(context.Background(), pidapi.PID{UniqID: "b"}, q); err != nil { + t.Fatal(err) + } + if !q.enabled.Load() { + t.Fatal("expected preemption to be enabled for created processors") + } +} + +// A preempted process goes behind other ready work: on a single worker a +// process that keeps yielding completes while a spinning process runs. +func TestPreemptedProcessYieldsWorkerToReadyProcesses(t *testing.T) { + lc, pingDone := newPingLifecycle() + sched := newPreemptTestScheduler(1, lc) + sched.Start() + defer testStopScheduler(sched) + + spin := &spinProcess{} + defer spin.release.Store(true) + if _, err := sched.Submit(context.Background(), pidapi.PID{UniqID: "spin"}, spin, "", nil); err != nil { + t.Fatal(err) + } + const rounds = 50 + if _, err := sched.Submit(context.Background(), pidapi.PID{UniqID: "ping"}, &pingProcess{rounds: rounds}, "", nil); err != nil { + t.Fatal(err) + } + + select { + case <-pingDone: + case <-time.After(5 * time.Second): + t.Fatalf("ready process starved by preempted process (%d spin steps)", spin.steps.Load()) + } +} + +// After a preempted step the worker serves its other ready work before the +// preempted process runs again. +func TestPreemptedProcessIsQueuedBehindLocalReadyWork(t *testing.T) { + sched := newPreemptTestScheduler(1, &testLifecycle{}) + spin := &spinProcess{} + spinProc, err := sched.Submit(context.Background(), pidapi.PID{UniqID: "spin"}, spin, "", nil) + if err != nil { + t.Fatal(err) + } + other := &spinProcess{} + otherProc, err := sched.Submit(context.Background(), pidapi.PID{UniqID: "other"}, other, "", nil) + if err != nil { + t.Fatal(err) + } + + w := sched.workerSnapshot()[0] + if got := w.findWork(); got != spinProc { + t.Fatalf("expected the first submission, got %p", got) + } + w.executeOne(spinProc) + if spin.steps.Load() != 1 { + t.Fatalf("expected one spin step, got %d", spin.steps.Load()) + } + if got := w.findWork(); got != otherProc { + t.Fatalf("expected ready work before the preempted process, got %p (spin %p)", got, spinProc) + } + if got := w.findWork(); got != spinProc { + t.Fatalf("expected the preempted process to be runnable again, got %p", got) + } +} + +func TestPreemptedProcessRunsAgainWithPendingYields(t *testing.T) { + done := make(chan struct{}) + lc := &testLifecycle{ + onComplete: func(context.Context, pidapi.PID, *runtime.Result) { close(done) }, + } + sched := newPreemptTestScheduler(1, lc) + sched.Start() + defer testStopScheduler(sched) + + p := &preemptWithPendingYieldProcess{} + if _, err := sched.Submit(context.Background(), testPID(), p, "", nil); err != nil { + t.Fatal(err) + } + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatalf("preempted process with a pending yield was not run again (%d steps)", p.steps.Load()) + } +} + +// upgradeSource requests an upgrade on its first step and reports a fixed +// step count to the scheduler. +type upgradeSource struct { + UpgradeProcess + used uint64 +} + +func (p *upgradeSource) StepsUsed() uint64 { return p.used } +func (p *upgradeSource) ResumeStepCount(n uint64) { p.used = n } + +// upgradeTarget records how the scheduler prepared it as an upgrade replacement. +type upgradeTarget struct { + stop chan struct{} + options atomic.Value + // counted closes when the scheduler calls ResumeStepCount, the last + // preparation step of an upgrade replacement. + counted chan struct{} + resumed atomic.Uint64 + enabled atomic.Bool + spinning bool +} + +func newUpgradeTarget() *upgradeTarget { + return &upgradeTarget{counted: make(chan struct{})} +} + +func (p *upgradeTarget) Init(ctx context.Context, _ string, _ payload.Payloads) error { + if opts := runtime.GetFrameLifecycleOptions(ctx); opts != nil { + p.options.Store(opts) + } + return nil +} +func (p *upgradeTarget) EnablePreemption() { p.enabled.Store(true) } +func (p *upgradeTarget) StepsUsed() uint64 { return p.resumed.Load() } +func (p *upgradeTarget) ResumeStepCount(used uint64) { + p.resumed.Store(used) + close(p.counted) +} +func (p *upgradeTarget) Step(_ []process.Event, out *process.StepOutput) error { + if p.spinning { + if p.enabled.Load() { + out.Preempt() + return nil + } + // Without preemption the step never ends on its own. + select { + case <-p.stop: + case <-time.After(10 * time.Second): + } + } + out.Done(nil) + return nil +} +func (*upgradeTarget) Send(*relay.Package) error { return nil } +func (*upgradeTarget) Close() {} + +// upgradeInto submits an upgrade source whose replacement is target and waits +// until the scheduler has initialized it and carried the step count over. +func upgradeInto(ctx context.Context, t *testing.T, sched *Scheduler, source process.Process, target *upgradeTarget) { + t.Helper() + appCtx := ctxapi.WithAppContext(ctx, ctxapi.NewAppContext()) + process.WithFactory(appCtx, &mockFactory{ + createFunc: func(registry.ID) (process.Process, *process.Meta, error) { + return target, &process.Meta{Method: "main"}, nil + }, + }) + frameCtx, fc := ctxapi.OpenFrameContext(appCtx) + fc.Seal() + if _, err := sched.Submit(frameCtx, pidapi.PID{UniqID: "upgrader"}, source, "", nil); err != nil { + t.Fatal(err) + } + select { + case <-target.counted: + case <-time.After(5 * time.Second): + t.Fatal("upgrade replacement was not initialized") + } +} + +func TestUpgradeReplacementIsPreemptionEnabled(t *testing.T) { + sched := newPreemptTestScheduler(1, &testLifecycle{}) + sched.Start() + defer testStopScheduler(sched) + + target := newUpgradeTarget() + source := &upgradeSource{UpgradeProcess: UpgradeProcess{upgradeReq: &process.UpgradeRequest{Source: registry.NewID("app", "next")}}} + upgradeInto(context.Background(), t, sched, source, target) + if !target.enabled.Load() { + t.Fatal("expected preemption to be enabled on the upgrade replacement") + } +} + +// An actor that upgrades into a spinning definition still yields its worker. +func TestUpgradeIntoSpinningActorYieldsWorker(t *testing.T) { + lc, pingDone := newPingLifecycle() + sched := newPreemptTestScheduler(1, lc) + sched.Start() + defer testStopScheduler(sched) + + target := newUpgradeTarget() + target.spinning, target.stop = true, make(chan struct{}) + defer close(target.stop) + source := &upgradeSource{UpgradeProcess: UpgradeProcess{upgradeReq: &process.UpgradeRequest{Source: registry.NewID("app", "next")}}} + upgradeInto(context.Background(), t, sched, source, target) + + if _, err := sched.Submit(context.Background(), pidapi.PID{UniqID: "ping"}, &pingProcess{rounds: 20}, "", nil); err != nil { + t.Fatal(err) + } + select { + case <-pingDone: + case <-time.After(5 * time.Second): + t.Fatal("ready process starved by the upgraded actor") + } +} + +func TestUpgradeKeepsSpawnLifecycleOptions(t *testing.T) { + sched := newPreemptTestScheduler(1, &testLifecycle{}) + sched.Start() + defer testStopScheduler(sched) + + spawn := attrs.Bag{"max_steps": int64(7)} + ctx, fc := ctxapi.OpenFrameContext(context.Background()) + if err := fc.Set(runtime.FrameLifecycleOptionsKey, attrs.Attributes(spawn)); err != nil { + t.Fatal(err) + } + target := newUpgradeTarget() + source := &upgradeSource{UpgradeProcess: UpgradeProcess{upgradeReq: &process.UpgradeRequest{Source: registry.NewID("app", "next")}}} + upgradeInto(ctx, t, sched, source, target) + + got, _ := target.options.Load().(attrs.Attributes) + if got == nil || got.GetInt("max_steps", 0) != 7 { + t.Fatalf("expected spawn options on the upgrade frame, got %v", got) + } +} + +func TestUpgradeCarriesStepCount(t *testing.T) { + sched := newPreemptTestScheduler(1, &testLifecycle{}) + sched.Start() + defer testStopScheduler(sched) + + target := newUpgradeTarget() + source := &upgradeSource{ + UpgradeProcess: UpgradeProcess{upgradeReq: &process.UpgradeRequest{Source: registry.NewID("app", "next")}}, + used: 41, + } + upgradeInto(context.Background(), t, sched, source, target) + if got := target.resumed.Load(); got != 41 { + t.Fatalf("expected 41 steps carried to the replacement, got %d", got) + } +} + +// A wake-up goes to a worker that is parked: a worker that is running a step +// cannot take the work before that step ends. +func TestWakeAnySignalsParkedWorker(t *testing.T) { + s := newPreemptTestScheduler(2, &testLifecycle{}) + busy, parked := newWorker(0, s), newWorker(1, s) + s.storeWorkers([]*Worker{busy, parked}) + parked.parked.Store(true) + + s.wakeAny() + + if busy.notified.Load() { + t.Fatal("a running worker was notified while another worker was parked") + } + if !parked.notified.Load() { + t.Fatal("the parked worker was not notified") + } +} diff --git a/system/scheduler/actor/processor.go b/system/scheduler/actor/processor.go index 8712a03e1..82b1c1d28 100644 --- a/system/scheduler/actor/processor.go +++ b/system/scheduler/actor/processor.go @@ -77,20 +77,25 @@ const ( // - StateBlocked: no owner, CompleteYield can CAS to Ready and re-queue // - Worker uses CAS loop to atomically check wakeup and transition state type Processor struct { - root context.Context - ctx context.Context - Process process.Process - stats atomic.Pointer[attrs.Bag] - resultCh chan *runtime.Result - scheduler *Scheduler - cancel context.CancelFunc - queue *process.EventQueue - sig atomic.Pointer[signalRef] + root context.Context + ctx context.Context + Process process.Process + stats atomic.Pointer[attrs.Bag] + resultCh chan *runtime.Result + scheduler *Scheduler + cancel context.CancelFunc + queue *process.EventQueue + sig atomic.Pointer[signalRef] + // completer receives yield completions for the current incarnation. It is + // bound to the queue generation, so completions that outlive the + // incarnation do not reach a later user of this pooled processor. + completer *process.YieldCompleter inspector atomic.Pointer[inspectorRef] pid pid.PID output process.StepOutput gen atomic.Uint64 steps atomic.Uint64 + wakeMu sync.Mutex id uint64 startedAt int64 state atomic.Int32 @@ -98,6 +103,40 @@ type Processor struct { pooled bool } +// setGeneration binds the processor to a new queue generation. A wake in +// progress completes against the incarnation it checked, so a wake bound to an +// earlier generation never transitions this one. +func (p *Processor) setGeneration(gen uint64) { + p.wakeMu.Lock() + p.gen.Store(gen) + p.wakeMu.Unlock() +} + +// casStateAt is casState for a wake bound to a queue generation: the +// transition happens only while the processor still runs that generation. +func (p *Processor) casStateAt(gen uint64, old, newState ProcessState) bool { + p.wakeMu.Lock() + defer p.wakeMu.Unlock() + return p.gen.Load() == gen && p.casState(old, newState) +} + +// wake makes a processor that waits runnable once an event was queued for it +// under gen, and reports whether the caller must schedule it. A running +// processor is flagged instead, and re-queues itself when it stops. The +// generation check and the transition are one step against slot reuse. +func (p *Processor) wake(gen uint64) bool { + p.wakeMu.Lock() + defer p.wakeMu.Unlock() + if p.gen.Load() != gen { + return false + } + if p.casState(StateBlocked, StateReady) || p.casState(StateIdle, StateReady) { + return true + } + p.setWakeup(StateRunning) + return false +} + // releaseIncarnation releases only frames created by an upgrade. The original // execution frame remains owned by the host that admitted the process. func (p *Processor) releaseIncarnation() { @@ -201,36 +240,6 @@ func (p *Processor) finishDispatch() bool { } } -// CompleteYield implements dispatcher.ResultReceiver. -// Called by handlers to deliver yield completion. -// Thread-safe: can be called from any goroutine. -func (p *Processor) CompleteYield(tag uint64, data any, err error) { - if !p.queue.Push(process.Event{ - Type: process.EventYieldComplete, - Tag: tag, - Data: data, - Error: err, - }, p.gen.Load()) { - return - } - - // Try to transition Blocked→Ready and re-queue. - // If processor is Running (worker still owns it), set wakeup flag atomically. - // Worker will check wakeup after dispatch and re-queue if set. - if p.casState(StateBlocked, StateReady) { - sched := p.scheduler - if sched != nil { - sched.injectOrGlobal(p) - } - return - } - - // Failed to transition - processor might be Running. - // Try to set wakeup flag atomically. If state changed, that's fine - - // either worker already moved on or another CompleteYield already woke it. - p.setWakeup(StateRunning) -} - // StateName returns a human-readable name for the process state. func StateName(s ProcessState) string { switch s & stateMask { @@ -257,6 +266,8 @@ var processorPool = sync.Pool{ queue: process.NewEventQueue(), } p.lastWorker.Store(noWorkerAffinity) + // A processor becomes Ready only when its submitter publishes it. + p.state.Store(int32(StateComplete)) return p }, } @@ -279,6 +290,7 @@ func releaseProcessor(p *Processor) { p.root = nil p.ctx = nil p.cancel = nil + p.completer = nil p.scheduler = nil p.lastWorker.Store(noWorkerAffinity) p.resultCh = nil diff --git a/system/scheduler/actor/scheduler.go b/system/scheduler/actor/scheduler.go index 2b291a686..df35137be 100644 --- a/system/scheduler/actor/scheduler.go +++ b/system/scheduler/actor/scheduler.go @@ -100,6 +100,8 @@ type Scheduler struct { registry dispatcher.Registry global *Queue drainCh chan struct{} + admitIdle *sync.Cond + admissionCancels map[uint64]context.CancelFunc byQueue sync.Map byPID sync.Map workers atomic.Pointer[workerSet] @@ -114,6 +116,12 @@ type Scheduler struct { retiredStolen atomic.Uint64 queueSize int nextID atomic.Uint64 + parked atomic.Int32 + // admitting counts Submit and CreateProcessor calls that have not + // finished publishing their processor; guarded by admitMu. + admitting int32 + nextAdmission uint64 + admitMu sync.Mutex phase atomic.Uint32 collectStats atomic.Bool started bool @@ -122,6 +130,60 @@ type Scheduler struct { func (s *Scheduler) isStopping() bool { return s.phase.Load() != phaseRunning } +// beginAdmission registers an admission and reports false once the scheduler +// is stopping. The check and the registration are one step, so Stop's +// awaitAdmissions sees every admission that was allowed to begin. +func (s *Scheduler) beginAdmission(cancel context.CancelFunc) (uint64, bool) { + s.admitMu.Lock() + defer s.admitMu.Unlock() + if s.isStopping() { + return 0, false + } + if s.admissionCancels == nil { + s.admissionCancels = make(map[uint64]context.CancelFunc) + } + s.nextAdmission++ + s.admissionCancels[s.nextAdmission] = cancel + s.admitting++ + return s.nextAdmission, true +} + +func (s *Scheduler) endAdmission(id uint64) { + s.admitMu.Lock() + delete(s.admissionCancels, id) + s.admitting-- + if s.admitting == 0 { + s.admitIdle.Broadcast() + } + s.admitMu.Unlock() +} + +// awaitAdmissions waits only within the shutdown budget. On expiry, cancel +// in-flight Init/OnStart calls without closing processes they still own. +// The notification is shutdown-only, not an allocation on every admission. +func (s *Scheduler) awaitAdmissions(ctx context.Context) { + stopWake := context.AfterFunc(ctx, func() { + s.admitMu.Lock() + s.admitIdle.Broadcast() + s.admitMu.Unlock() + }) + defer stopWake() + s.admitMu.Lock() + for s.admitting > 0 && ctx.Err() == nil { + s.admitIdle.Wait() + } + var cancels []context.CancelFunc + if ctx.Err() != nil { + for _, cancel := range s.admissionCancels { + cancels = append(cancels, cancel) + } + } + s.admitMu.Unlock() + for _, cancel := range cancels { + cancel() + } +} + func NewScheduler(registry dispatcher.Registry, opts ...Option) *Scheduler { s := &Scheduler{ registry: registry, @@ -136,6 +198,7 @@ func NewScheduler(registry dispatcher.Registry, opts ...Option) *Scheduler { s.global = NewQueue(s.queueSize) s.drainCh = make(chan struct{}, 1) + s.admitIdle = sync.NewCond(&s.admitMu) workers := make([]*Worker, s.initialWorkers) for i := range workers { workers[i] = newWorker(i, s) @@ -152,20 +215,33 @@ func (s *Scheduler) getHandler(cmd dispatcher.Command) dispatcher.Handler { // Stop gracefully shuts down the scheduler. // Sends cancel events and waits for processes to complete or context deadline. func (s *Scheduler) Stop(ctx context.Context) { + waitCtx := ctx + if _, ok := ctx.Deadline(); !ok { + var cancel context.CancelFunc + waitCtx, cancel = context.WithTimeout(ctx, 10*time.Second) + defer cancel() + } // Begin draining while serialized with Start and Resize, then // release the control lock before lifecycle callbacks and worker waits. s.controlMu.Lock() + s.admitMu.Lock() if !s.phase.CompareAndSwap(phaseRunning, phaseDraining) { + s.admitMu.Unlock() s.controlMu.Unlock() return } + s.admitMu.Unlock() s.controlMu.Unlock() + s.awaitAdmissions(waitCtx) // Push cancel event directly to each processor's queue. // Safe because draining prevents pool release. // Wake idle/blocked processors so they process the cancel. s.byPID.Range(func(_, value any) bool { proc := value.(*Processor) + if ProcessState(proc.state.Load())&stateMask == StateComplete { + return true // An unfinished admission owns its process until OnStart returns. + } pkg := topology.CancelPackage(pid.PID{}, proc.pid, "scheduler shutdown") proc.queue.PushDirect(process.Event{ Type: process.EventMessage, @@ -190,22 +266,20 @@ func (s *Scheduler) Stop(ctx context.Context) { } // Wait for processes to complete or context timeout - waitCtx := ctx - if _, ok := ctx.Deadline(); !ok { - var cancel context.CancelFunc - waitCtx, cancel = context.WithTimeout(ctx, 10*time.Second) - defer cancel() - } - select { case <-s.drainCh: // All processes completed gracefully case <-waitCtx.Done(): - // Timeout - cancel all process contexts to unblock stuck processes + // Timeout - cancel all process contexts to unblock stuck processes. + // A processor may be starting or completing concurrently, so act + // only through its published incarnation, as Terminate does. s.byPID.Range(func(_, value any) bool { proc := value.(*Processor) - if proc.cancel != nil { - proc.cancel() + if ProcessState(proc.state.Load())&stateMask == StateComplete { + return true + } + if ref := proc.sig.Load(); ref != nil { + ref.terminate() } proc.queue.Close() return true @@ -215,26 +289,42 @@ func (s *Scheduler) Stop(ctx context.Context) { // Wake and wait for workers to exit // Admission closes before cancellation, but workers must remain available // for asynchronous cleanup until processes drain or the deadline expires. + s.admitMu.Lock() s.phase.CompareAndSwap(phaseDraining, phaseStoppingWorkers) + s.admitMu.Unlock() s.wakeAll() s.wg.Wait() // Force complete any remaining processes (workers stopped) s.byPID.Range(func(_, value any) bool { proc := value.(*Processor) - s.completeNoPool(proc, nil, context.Canceled) + if ProcessState(proc.state.Load())&stateMask != StateComplete { + s.completeNoPool(proc, nil, context.Canceled) + } return true }) } +// wakeAny notifies one parked worker. A worker that is not parked looks for +// work when its current step ends, so notifying it cannot start the work +// while that step runs; the park protocol guarantees that a worker about to +// park either sees the queued work or is seen as parked. func (s *Scheduler) wakeAny() { for _, w := range s.workerSnapshot() { - if w.signal() { + if w.parked.Load() && w.signal() { return } } } +// enablePreemption lets a preemptible process suspend long steps: the +// scheduler runs it again after StepPreempted. +func enablePreemption(p process.Process) { + if pp, ok := p.(process.Preemptible); ok { + pp.EnablePreemption() + } +} + func (s *Scheduler) wakeAll() { for _, w := range s.workerSnapshot() { _ = w.signal() @@ -272,21 +362,9 @@ func (s *Scheduler) WakeProcessor(q *process.EventQueue, gen uint64) { } proc := v.(*Processor) - // Verify generation matches to avoid waking wrong processor - if proc.gen.Load() != gen { - return - } - - // Same wake logic as processor.CompleteYield - if proc.casState(StateBlocked, StateReady) { + if proc.wake(gen) { s.injectOrGlobal(proc) - return } - if proc.casState(StateIdle, StateReady) { - s.injectOrGlobal(proc) - return - } - proc.setWakeup(StateRunning) } func (s *Scheduler) Submit(ctx context.Context, pid pid.PID, p process.Process, method string, input payload.Payloads) (*Processor, error) { @@ -313,17 +391,27 @@ func (s *Scheduler) Submit(ctx context.Context, pid pid.PID, p process.Process, } else { procCtx, cancel = context.WithCancel(ctx) } + admission, ok := s.beginAdmission(cancel) + if !ok { + cancel() + return nil, process.ErrSchedulerStopping + } + defer s.endAdmission(admission) + enablePreemption(p) if err := p.Init(procCtx, method, input); err != nil { cancel() return nil, err } + if err := procCtx.Err(); err != nil { + cancel() + return nil, err + } proc := acquireProcessor() proc.id = s.nextID.Add(1) proc.pid = pid proc.Process = p - proc.state.Store(int32(StateReady)) proc.root = procCtx proc.ctx = procCtx proc.cancel = cancel @@ -335,7 +423,8 @@ func (s *Scheduler) Submit(ctx context.Context, pid pid.PID, p process.Process, if admission, ok := p.(interface{ EventAdmission() process.EventAdmission }); ok { proc.queue.SetAdmission(admission.EventAdmission()) } - proc.gen.Store(proc.queue.Generation()) + proc.setGeneration(proc.queue.Generation()) + proc.completer = proc.queue.NewYieldCompleter(s) proc.publishSignalRef(cancel) proc.publishInspectorRef() @@ -346,30 +435,71 @@ func (s *Scheduler) Submit(ctx context.Context, pid pid.PID, p process.Process, if s.lifecycle != nil { if err := s.lifecycle.OnStart(procCtx, pid, p); err != nil { // Rollback: routing was ready but lifecycle rejected - s.byPID.Delete(pid.String()) - s.byQueue.Delete(proc.queue) - s.processorCount.Add(-1) - cancel() - p.Close() - releaseProcessor(proc) + s.rejectAdmission(proc) return nil, err } } - // Cancellation must also wake an actor parked without incoming messages. - // Capture only queue identity/generation: Processor objects are pooled. + s.wakeOnCancel(procCtx, proc, cancel) + + // Serialize publication with worker shutdown. A callback that ignores + // cancellation may return after Stop, but cannot enqueue work on dead workers. + s.admitMu.Lock() + if s.phase.Load() == phaseStoppingWorkers { + s.admitMu.Unlock() + proc.queue.Close() + s.completeNoPool(proc, nil, context.Canceled) + return proc, nil // Successful OnStart transfers cleanup to the scheduler. + } + // A pooled processor can still be referenced by a stale queue entry from + // its previous incarnation; Ready is published only once it is initialized. + proc.state.Store(int32(StateReady)) + s.global.Push(proc) + s.admitMu.Unlock() + s.wakeAny() + + return proc, nil +} + +// A rejected admission never ran and still belongs to its caller. Remove its +// routing without Close; don't recycle a slot while shutdown may be scanning it. +func (s *Scheduler) rejectAdmission(proc *Processor) { + s.byPID.Delete(proc.pid.String()) + s.byQueue.Delete(proc.queue) + s.uncount() + proc.cancel() + proc.queue.Close() + proc.sig.Store(nil) + proc.inspector.Store(nil) + s.admitMu.Lock() + if !s.isStopping() { + releaseProcessor(proc) + } + s.admitMu.Unlock() +} + +// wakeOnCancel makes cancellation of the incarnation's context wake an actor +// parked without incoming messages. It captures only queue identity and +// generation, because Processor objects are pooled. +func (s *Scheduler) wakeOnCancel(ctx context.Context, proc *Processor, cancel context.CancelFunc) { q, gen := proc.queue, proc.gen.Load() - stopWake := context.AfterFunc(procCtx, func() { + stopWake := context.AfterFunc(ctx, func() { if q.Push(process.Event{Type: process.EventMessage}, gen) { s.WakeProcessor(q, gen) } }) proc.cancel = func() { stopWake(); cancel() } +} - s.global.Push(proc) - s.wakeAny() - - return proc, nil +// uncount removes a processor from the live count and tells a draining Stop +// when none remain. +func (s *Scheduler) uncount() { + if s.processorCount.Add(-1) == 0 && s.isStopping() { + select { + case s.drainCh <- struct{}{}: + default: + } + } } // Terminate forcibly terminates a process by PID. @@ -410,12 +540,7 @@ func (s *Scheduler) finishProcessor(proc *Processor, result *process.StepOutput, stopping := s.isStopping() if !proc.pooled { - if s.processorCount.Add(-1) == 0 && stopping { - select { - case s.drainCh <- struct{}{}: - default: - } - } + s.uncount() } if proc.resultCh != nil { @@ -452,14 +577,21 @@ func (s *Scheduler) CreateProcessor(ctx context.Context, pid pid.PID, p process. return nil, process.ErrMaxProcessesExceeded } + enablePreemption(p) + // Wrap context with cancel for Terminate support procCtx, cancel := context.WithCancel(ctx) + admission, ok := s.beginAdmission(cancel) + if !ok { + cancel() + return nil, process.ErrSchedulerStopping + } + defer s.endAdmission(admission) proc := acquireProcessor() proc.id = s.nextID.Add(1) proc.pid = pid proc.Process = p - proc.state.Store(int32(StateReady)) proc.root = procCtx proc.ctx = procCtx proc.cancel = cancel @@ -470,19 +602,29 @@ func (s *Scheduler) CreateProcessor(ctx context.Context, pid pid.PID, p process. // Reset queue for this execution and cache generation proc.queue.Reset() - proc.gen.Store(proc.queue.Generation()) + proc.setGeneration(proc.queue.Generation()) + proc.completer = proc.queue.NewYieldCompleter(s) proc.publishSignalRef(cancel) proc.publishInspectorRef() s.processorCount.Add(1) s.byPID.Store(pid.String(), proc) s.byQueue.Store(proc.queue, proc) + s.wakeOnCancel(procCtx, proc, cancel) + s.admitMu.Lock() + if s.phase.Load() == phaseStoppingWorkers { + s.admitMu.Unlock() + s.rejectAdmission(proc) + return nil, process.ErrSchedulerStopping + } + // Ready is published only once the processor is initialized; see Submit. + proc.state.Store(int32(StateReady)) + s.admitMu.Unlock() return proc, nil } func (s *Scheduler) ReleaseProcessor(proc *Processor) { - s.processorCount.Add(-1) proc.sig.Store(nil) proc.inspector.Store(nil) s.byPID.Delete(proc.pid.String()) @@ -493,6 +635,7 @@ func (s *Scheduler) ReleaseProcessor(proc *Processor) { if proc.Process != nil { proc.Process.Close() } + s.uncount() proc.releaseIncarnation() } @@ -599,9 +742,7 @@ func (s *Scheduler) deliverToProcError(proc *Processor, gen uint64, pkg *relay.P // Wake process if waiting for messages. // CAS ensures exactly-once wake even with concurrent senders. // Try both Idle (waiting on select) and Blocked (waiting on yield completion). - if proc.casState(StateIdle, StateReady) { - s.injectOrGlobal(proc) - } else if proc.casState(StateBlocked, StateReady) { + if proc.casStateAt(gen, StateIdle, StateReady) || proc.casStateAt(gen, StateBlocked, StateReady) { s.injectOrGlobal(proc) } diff --git a/system/scheduler/actor/stop_admission_test.go b/system/scheduler/actor/stop_admission_test.go new file mode 100644 index 000000000..da03fa8fb --- /dev/null +++ b/system/scheduler/actor/stop_admission_test.go @@ -0,0 +1,263 @@ +// SPDX-License-Identifier: MPL-2.0 + +package actor + +import ( + "context" + "errors" + "slices" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/wippyai/runtime/api/payload" + pidapi "github.com/wippyai/runtime/api/pid" + "github.com/wippyai/runtime/api/process" + "github.com/wippyai/runtime/api/runtime" +) + +// Shutdown is bounded, but never closes a process while OnStart still owns it. +// A successful late admission pairs OnStart with OnComplete without executing. +func TestStopDeadlineDuringLifecycleAdmission(t *testing.T) { + var mu sync.Mutex + var events []string + record := func(e string) { + mu.Lock() + events = append(events, e) + mu.Unlock() + } + entered, release := make(chan struct{}), make(chan struct{}) + lc := &testLifecycle{ + onStart: func(context.Context, pidapi.PID, process.Process) { + close(entered) + <-release + record("start returned") + }, + onComplete: func(context.Context, pidapi.PID, *runtime.Result) { record("complete") }, + } + sched := newPreemptTestScheduler(1, lc) + sched.Start() + + p := &closeCountingProcess{} + submitted := make(chan error, 1) + go func() { + _, err := sched.Submit(context.Background(), pidapi.PID{UniqID: "admitting"}, p, "", nil) + submitted <- err + }() + <-entered + + stopped := make(chan struct{}) + go func() { + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + sched.Stop(ctx) + close(stopped) + }() + select { + case <-stopped: + case <-time.After(600 * time.Millisecond): + t.Error("Stop ignored its deadline during OnStart") + } + if got := p.closed.Load(); got != 0 { + t.Errorf("closed process while OnStart was running: %d", got) + } + mu.Lock() + if len(events) != 0 { + t.Errorf("completion before OnStart returned: %v", events) + } + mu.Unlock() + close(release) + + if err := <-submitted; err != nil { + t.Fatalf("submit: %v", err) + } + <-stopped + + mu.Lock() + defer mu.Unlock() + if want := []string{"start returned", "complete"}; !slices.Equal(events, want) { + t.Fatalf("lifecycle events %v, want %v", events, want) + } + if got := p.closed.Load(); got != 1 { + t.Fatalf("process closed %d times", got) + } +} + +type blockedAdmissionProcess struct { + entered chan struct{} + release chan struct{} + cooperative bool + closed atomic.Int32 + stepped atomic.Int32 +} + +func (p *blockedAdmissionProcess) Init(ctx context.Context, _ string, _ payload.Payloads) error { + close(p.entered) + if p.cooperative { + <-ctx.Done() + return ctx.Err() + } + <-p.release + return nil +} + +func (p *blockedAdmissionProcess) Step(_ []process.Event, out *process.StepOutput) error { + p.stepped.Add(1) + out.Done(nil) + return nil +} + +func (p *blockedAdmissionProcess) Close() { p.closed.Add(1) } + +func TestStopDeadlineDuringInit(t *testing.T) { + for _, cooperative := range []bool{true, false} { + name := "ignores-cancel" + if cooperative { + name = "cooperative" + } + t.Run(name, func(t *testing.T) { + var started, completed atomic.Int32 + sched := newPreemptTestScheduler(1, &testLifecycle{ + onStart: func(context.Context, pidapi.PID, process.Process) { started.Add(1) }, + onComplete: func(context.Context, pidapi.PID, *runtime.Result) { completed.Add(1) }, + }) + sched.Start() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + p := &blockedAdmissionProcess{entered: make(chan struct{}), release: make(chan struct{}), cooperative: cooperative} + submitted := make(chan error, 1) + go func() { _, err := sched.Submit(ctx, pidapi.PID{UniqID: name}, p, "", nil); submitted <- err }() + <-p.entered + stopped := make(chan struct{}) + go func() { + stopCtx, stopCancel := context.WithTimeout(context.Background(), 40*time.Millisecond) + defer stopCancel() + sched.Stop(stopCtx) + close(stopped) + }() + select { + case <-stopped: + case <-time.After(500 * time.Millisecond): + t.Error("Stop ignored its deadline during Init") + } + if got := p.closed.Load(); got != 0 { + t.Errorf("scheduler closed caller-owned process: %d", got) + } + close(p.release) + cancel() // Also unblocks the broken implementation on regression failure. + if err := <-submitted; !errors.Is(err, context.Canceled) { + t.Errorf("Submit = %v, want canceled admission", err) + } + <-stopped + if started.Load() != 0 || completed.Load() != 0 || p.stepped.Load() != 0 { + t.Errorf("canceled Init ran lifecycle or code: start=%d complete=%d step=%d", started.Load(), completed.Load(), p.stepped.Load()) + } + if got := sched.processorCount.Load(); got != 0 { + t.Errorf("leaked %d processors", got) + } + p.Close() // Failed Submit leaves cleanup with the caller. + }) + } +} + +func TestStopCancelsLifecycleAdmission(t *testing.T) { + entered := make(chan struct{}) + var completed atomic.Int32 + sched := newPreemptTestScheduler(1, &testLifecycle{ + onStart: func(ctx context.Context, _ pidapi.PID, _ process.Process) { close(entered); <-ctx.Done() }, + onComplete: func(context.Context, pidapi.PID, *runtime.Result) { completed.Add(1) }, + }) + sched.Start() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + p := &closeCountingProcess{} + submitted := make(chan error, 1) + go func() { _, err := sched.Submit(ctx, pidapi.PID{UniqID: "cancel-start"}, p, "", nil); submitted <- err }() + <-entered + stopped := make(chan struct{}) + go func() { + stopCtx, stopCancel := context.WithTimeout(context.Background(), 40*time.Millisecond) + defer stopCancel() + sched.Stop(stopCtx) + close(stopped) + }() + select { + case <-stopped: + case <-time.After(500 * time.Millisecond): + t.Error("Stop did not cancel OnStart") + cancel() + } + if err := <-submitted; err != nil { + t.Fatalf("Submit: %v", err) + } + <-stopped + if completed.Load() != 1 || p.closed.Load() != 1 { + t.Fatalf("completion=%d close=%d; want one of each", completed.Load(), p.closed.Load()) + } +} + +type rejectingAdmission struct { + testLifecycle + entered chan struct{} + release chan struct{} + err error +} + +func (l *rejectingAdmission) OnStart(context.Context, pidapi.PID, process.Process) error { + close(l.entered) + <-l.release + return l.err +} + +func TestLateRejectedAdmissionKeepsCallerOwnership(t *testing.T) { + var completed atomic.Int32 + lc := &rejectingAdmission{ + entered: make(chan struct{}), release: make(chan struct{}), err: errors.New("rejected"), + testLifecycle: testLifecycle{onComplete: func(context.Context, pidapi.PID, *runtime.Result) { completed.Add(1) }}, + } + sched := newPreemptTestScheduler(1, &lc.testLifecycle) + sched.lifecycle = lc + sched.Start() + p := &closeCountingProcess{} + self := pidapi.PID{UniqID: "late-rejection"} + submitted := make(chan error, 1) + go func() { + _, err := sched.Submit(context.Background(), self, p, "", nil) + submitted <- err + }() + <-lc.entered + stopped := make(chan struct{}) + go func() { + ctx, cancel := context.WithTimeout(context.Background(), 40*time.Millisecond) + defer cancel() + sched.Stop(ctx) + close(stopped) + }() + select { + case <-stopped: + case <-time.After(500 * time.Millisecond): + t.Error("Stop ignored its deadline during rejected admission") + } + close(lc.release) + if err := <-submitted; !errors.Is(err, lc.err) { + t.Errorf("Submit=%v; want rejection", err) + } + <-stopped + if p.closed.Load() != 0 || completed.Load() != 0 { + t.Fatal("rejected process was completed or closed by scheduler") + } + if sched.processorCount.Load() != 0 { + t.Fatal("rejected admission retained process count") + } + if _, ok := sched.byPID.Load(self.String()); ok { + t.Fatal("rejected admission retained routing") + } + sched.admitMu.Lock() + pending := len(sched.admissionCancels) + sched.admitMu.Unlock() + if pending != 0 { + t.Fatal("rejected admission retained its cancel") + } + p.Close() +} diff --git a/system/scheduler/actor/submit_publication_test.go b/system/scheduler/actor/submit_publication_test.go new file mode 100644 index 000000000..1a20dd117 --- /dev/null +++ b/system/scheduler/actor/submit_publication_test.go @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: MPL-2.0 + +package actor + +import ( + "context" + "sync/atomic" + "testing" + + "github.com/wippyai/runtime/api/payload" + pidapi "github.com/wippyai/runtime/api/pid" + "github.com/wippyai/runtime/api/process" + "github.com/wippyai/runtime/api/relay" +) + +// stepRecorder counts the steps it runs. +type stepRecorder struct{ steps atomic.Int32 } + +func (*stepRecorder) Init(context.Context, string, payload.Payloads) error { return nil } + +func (p *stepRecorder) Step(_ []process.Event, out *process.StepOutput) error { + p.steps.Add(1) + out.Done(nil) + return nil +} + +func (*stepRecorder) Send(*relay.Package) error { return nil } +func (*stepRecorder) Close() {} + +// A pooled processor can still sit in a worker's queue from its previous +// incarnation. Until Submit has initialized it, popping that stale entry +// must not run the new process. +func TestSubmitPublishesProcessorOnlyWhenInitialized(t *testing.T) { + proc := &stepRecorder{} + self := pidapi.PID{UniqID: "half-initialized"} + var sched *Scheduler + var ranEarly int32 + sched = newTestSchedulerWithLifecycle(1, &testLifecycle{ + onStart: func(context.Context, pidapi.PID, process.Process) { + v, ok := sched.byPID.Load(self.String()) + if !ok { + t.Error("the processor is registered before lifecycle start") + return + } + // A worker pops the stale entry while Submit is still running. + sched.workerSnapshot()[0].executeOne(v.(*Processor)) + ranEarly = proc.steps.Load() + }, + }) + sched.Start() + defer testStopScheduler(sched) + + if _, err := sched.Submit(context.Background(), self, proc, "", nil); err != nil { + t.Fatal(err) + } + if ranEarly != 0 { + t.Fatalf("a stale queue entry ran the process before Submit initialized it (%d steps)", ranEarly) + } +} diff --git a/system/scheduler/actor/upgrade_execution_test.go b/system/scheduler/actor/upgrade_execution_test.go index 1815726db..3b6b1b89c 100644 --- a/system/scheduler/actor/upgrade_execution_test.go +++ b/system/scheduler/actor/upgrade_execution_test.go @@ -182,8 +182,15 @@ func TestUpgradeContinuesTheExecution(t *testing.T) { type failingIncarnation struct { incarnation - resource *countingCloser - err error + resource *countingCloser + err error + closed atomic.Int32 + closedBeforeFrame atomic.Bool +} + +func (p *failingIncarnation) Close() { + p.closed.Add(1) + p.closedBeforeFrame.Store(p.resource.closed.Load() == 0) } func (p *failingIncarnation) Init(ctx context.Context, _ string, _ payload.Payloads) error { @@ -196,6 +203,7 @@ func (p *failingIncarnation) Init(ctx context.Context, _ string, _ payload.Paylo func TestFailedUpgradeReleasesTheNewFrameAndExecution(t *testing.T) { wantErr := errors.New("replacement init failed") rootResource, leafResource := newCountingCloser(), newCountingCloser() + replacement := &failingIncarnation{resource: leafResource, err: wantErr} completed := make(chan *runtime.Result, 1) sched := newTestSchedulerWithLifecycle(1, &testLifecycle{ onComplete: func(ctx context.Context, _ pidapi.PID, result *runtime.Result) { @@ -208,7 +216,7 @@ func TestFailedUpgradeReleasesTheNewFrameAndExecution(t *testing.T) { appCtx := ctxapi.WithAppContext(context.Background(), ctxapi.NewAppContext()) process.WithFactory(appCtx, &mockFactory{ createFunc: func(registry.ID) (process.Process, *process.Meta, error) { - return &failingIncarnation{resource: leafResource, err: wantErr}, &process.Meta{}, nil + return replacement, &process.Meta{}, nil }, }) ctx, frame := ctxapi.OpenFrameContext(appCtx) @@ -239,4 +247,10 @@ func TestFailedUpgradeReleasesTheNewFrameAndExecution(t *testing.T) { t.Fatalf("each resource must close once, got %d", got) } } + if got := replacement.closed.Load(); got != 1 { + t.Errorf("replacement closed %d times; want exactly once", got) + } + if !replacement.closedBeforeFrame.Load() { + t.Error("replacement closed after its frame-owned resource was released") + } } diff --git a/system/scheduler/actor/upgrade_yield_test.go b/system/scheduler/actor/upgrade_yield_test.go new file mode 100644 index 000000000..eff117a6a --- /dev/null +++ b/system/scheduler/actor/upgrade_yield_test.go @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: MPL-2.0 + +package actor + +import ( + "context" + "testing" + "time" + + ctxapi "github.com/wippyai/runtime/api/context" + "github.com/wippyai/runtime/api/dispatcher" + "github.com/wippyai/runtime/api/payload" + pidapi "github.com/wippyai/runtime/api/pid" + "github.com/wippyai/runtime/api/process" + "github.com/wippyai/runtime/api/registry" + "github.com/wippyai/runtime/api/relay" + "github.com/wippyai/runtime/system/scheduler" +) + +// yieldingProcess yields tag 1 on its first step. Its second step either +// requests an upgrade or, when it has no request, records yield completions. +type yieldingProcess struct { + upgrade *process.UpgradeRequest + completed chan any + steps int +} + +func (*yieldingProcess) Init(context.Context, string, payload.Payloads) error { return nil } +func (p *yieldingProcess) Step(events []process.Event, out *process.StepOutput) error { + p.steps++ + if p.steps == 1 { + out.Yield(YieldCmd{}, 1) + out.WaitForYields() + return nil + } + for _, e := range events { + if e.Type == process.EventYieldComplete { + p.completed <- e.Data + out.Done(nil) + return nil + } + } + if p.upgrade != nil && len(events) > 0 { + out.SetUpgrade(p.upgrade) + return nil + } + out.WaitForYields() + return nil +} +func (*yieldingProcess) Send(*relay.Package) error { return nil } +func (*yieldingProcess) Close() {} + +// A yield completion of the replaced code never reaches the replacement, even +// when its tag matches a yield of the replacement; the replacement's own +// completion is delivered. +func TestUpgradeRetiresYieldCompletionsOfReplacedCode(t *testing.T) { + receivers := make(chan dispatcher.ResultReceiver, 2) + reg := scheduler.NewRegistry() + reg.Register(CmdYield, dispatcher.HandlerFunc(func(_ context.Context, _ dispatcher.Command, _ uint64, r dispatcher.ResultReceiver) error { + receivers <- r + return nil + })) + reg.Register(CmdComplete, CompleteHandler()) + sched := NewScheduler(reg, WithWorkers(1), WithLifecycle(&testLifecycle{})) + sched.Start() + defer testStopScheduler(sched) + + completed := make(chan any, 2) + next := registry.NewID("app", "next") + factory := &mockFactory{createFunc: func(registry.ID) (process.Process, *process.Meta, error) { + return &yieldingProcess{completed: completed}, &process.Meta{Method: "main"}, nil + }} + appCtx := ctxapi.WithAppContext(context.Background(), ctxapi.NewAppContext()) + process.WithFactory(appCtx, factory) + frameCtx, fc := ctxapi.OpenFrameContext(appCtx) + fc.Seal() + id := pidapi.PID{UniqID: "yielder"} + source := &yieldingProcess{upgrade: &process.UpgradeRequest{Source: next}, completed: completed} + if _, err := sched.Submit(frameCtx, id, source, "", nil); err != nil { + t.Fatal(err) + } + oldReceiver := receive(t, receivers, "source yield") + if err := sched.Send(&relay.Package{Target: id}); err != nil { + t.Fatal(err) + } + newReceiver := receive(t, receivers, "replacement yield") + + oldReceiver.CompleteYield(1, "stale", nil) + newReceiver.CompleteYield(1, "fresh", nil) + + select { + case got := <-completed: + if got != "fresh" { + t.Fatalf("replacement received %v, want its own completion", got) + } + case <-time.After(5 * time.Second): + t.Fatal("replacement did not receive its completion") + } +} + +func receive(t *testing.T, ch <-chan dispatcher.ResultReceiver, what string) dispatcher.ResultReceiver { + t.Helper() + select { + case r := <-ch: + return r + case <-time.After(5 * time.Second): + t.Fatalf("%s was not dispatched", what) + return nil + } +} diff --git a/system/scheduler/actor/wake_reuse_test.go b/system/scheduler/actor/wake_reuse_test.go new file mode 100644 index 000000000..9df70b9c0 --- /dev/null +++ b/system/scheduler/actor/wake_reuse_test.go @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: MPL-2.0 + +package actor + +import ( + "sync" + "testing" + + "github.com/wippyai/runtime/api/process" +) + +// A wake bound to an earlier incarnation of a processor slot never changes the +// state of the incarnation that reuses the slot, whichever way the reuse +// interleaves with the wake. +func TestStaleWakeDoesNotTransitionReusedSlot(t *testing.T) { + s := NewScheduler(nil) + proc := &Processor{queue: process.NewEventQueue()} + s.byQueue.Store(proc.queue, proc) + + const rounds = 20000 + for i := 0; i < rounds; i++ { + oldGen := uint64(2*i + 1) + proc.gen.Store(oldGen) + proc.state.Store(int32(StateBlocked)) + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + s.WakeProcessor(proc.queue, oldGen) + }() + // The slot is reused: the next incarnation publishes its generation and + // blocks. + proc.setGeneration(oldGen + 1) + proc.state.Store(int32(StateBlocked)) + wg.Wait() + + if got := ProcessState(proc.state.Load()) & stateMask; got != StateBlocked { + t.Fatalf("round %d: stale wake moved the reused slot to %s", i, StateName(got)) + } + for s.global.Pop() != nil { + continue + } + } +} + +func TestStaleWakeLeavesRunningSlotUnflagged(t *testing.T) { + proc := &Processor{queue: process.NewEventQueue()} + proc.setGeneration(2) + proc.state.Store(int32(StateRunning)) + + if proc.wake(1) { + t.Fatal("a stale wake must not schedule the processor") + } + if got := ProcessState(proc.state.Load()); got != StateRunning { + t.Fatalf("stale wake changed the state to %d", got) + } + if proc.wake(2); ProcessState(proc.state.Load()) != StateRunning|wakeupFlag { + t.Fatalf("a current wake flags the running processor, state %d", proc.state.Load()) + } +} diff --git a/system/scheduler/actor/worker.go b/system/scheduler/actor/worker.go index 0f38d78f5..0a4d23257 100644 --- a/system/scheduler/actor/worker.go +++ b/system/scheduler/actor/worker.go @@ -29,8 +29,9 @@ type Worker struct { executed atomic.Uint64 stolen atomic.Uint64 notified atomic.Bool - executing bool // guarded by routeMu retiring atomic.Bool + parked atomic.Bool // set while counted in Scheduler.parked + executing bool // guarded by routeMu dispatchesSinceFair uint8 fairSource uint8 } @@ -163,15 +164,46 @@ func (w *Worker) handoff(proc *Processor) { w.scheduler.global.Push(proc) } +// requeuePreempted makes a preempted processor runnable behind other ready +// work. When this worker has nothing else to run the processor continues on +// the local deque like StepContinue; otherwise it joins the global FIFO, where +// any worker can pick it up after the work already queued. The wakeup is only +// needed when some worker is parked: this worker looks for work again right +// after the step. +func (w *Worker) requeuePreempted(proc *Processor) { + s := w.scheduler + if w.local.IsEmpty() && w.inject.IsEmpty() && s.global.Len() == 0 { + w.local.Push(proc) + return + } + proc.lastWorker.Store(noWorkerAffinity) + s.global.Push(proc) + if s.parked.Load() > 0 { + s.wakeAny() + } +} + +func (w *Worker) unpark() { + w.parked.Store(false) + w.scheduler.parked.Add(-1) +} + func (w *Worker) park() { s := w.scheduler + // A worker counts as parked from before its last work check until it + // stops waiting, so a producer that publishes work and then reads the + // count either sees the worker or the worker sees the work. + s.parked.Add(1) + w.parked.Store(true) w.parkMu.Lock() for { if w.retiring.Load() { + w.unpark() w.parkMu.Unlock() return } if proc := w.findWork(); proc != nil { + w.unpark() shouldWake := s.global.Len() > 0 w.parkMu.Unlock() if shouldWake { @@ -182,6 +214,7 @@ func (w *Worker) park() { return } if s.phase.Load() == phaseStoppingWorkers { + w.unpark() w.parkMu.Unlock() return } @@ -220,7 +253,7 @@ func (w *Worker) signal() bool { // localDispatchQuantum bounds preference for the cache-hot local continuation. // Fair turns rotate through injected wakeups, global submissions and the oldest -// local continuation. This is cooperative dispatch fairness, not guest preemption. +// local continuation. const localDispatchQuantum = 32 func (w *Worker) takeFairWork() *Processor { @@ -398,7 +431,7 @@ func (w *Worker) executeOne(proc *Processor) { if status != process.StepDone { yields := proc.output.Yields() if len(yields) > 0 { - w.dispatchYields(proc.ctx, proc, yields) + w.dispatchYields(proc.ctx, proc, yields, status == process.StepPreempted) return } } @@ -420,6 +453,12 @@ func (w *Worker) executeOne(proc *Processor) { // No wake needed since we're the active worker. w.local.Push(proc) + case process.StepPreempted: + if !proc.casState(StateRunning, StateReady) { + return + } + w.requeuePreempted(proc) + case process.StepYield: if !proc.casState(StateRunning, StateBlocked) { return @@ -443,7 +482,6 @@ func (w *Worker) executeOne(proc *Processor) { case process.StepUpgrade: req := proc.output.Upgrade() if req == nil { - proc.Process.Close() if !proc.casState(StateRunning, StateComplete) { return } @@ -454,7 +492,6 @@ func (w *Worker) executeOne(proc *Processor) { factory := process.GetFactory(proc.ctx) if factory == nil { - proc.Process.Close() if !proc.casState(StateRunning, StateComplete) { return } @@ -469,7 +506,6 @@ func (w *Worker) executeOne(proc *Processor) { var ok bool source, ok = runtime.GetFrameID(proc.ctx) if !ok { - proc.Process.Close() if !proc.casState(StateRunning, StateComplete) { return } @@ -482,7 +518,6 @@ func (w *Worker) executeOne(proc *Processor) { // Create new process newProc, meta, err := factory.Create(source) if err != nil { - proc.Process.Close() if !proc.casState(StateRunning, StateComplete) { return } @@ -491,11 +526,22 @@ func (w *Worker) executeOne(proc *Processor) { return } + var stepsUsed uint64 + if counter, ok := proc.Process.(process.StepAccounted); ok { + stepsUsed = counter.StepsUsed() + } + // Close old process proc.Process.Close() // Swap proc.Process = newProc + enablePreemption(newProc) + // The replacement numbers its yields from the start; completions of the + // replaced code must not resume them. Messages and wakes stay bound to + // the unchanged queue generation. + proc.queue.RetireYieldCompletions() + proc.completer = proc.queue.NewYieldCompleter(w.scheduler) // Init new process method := "main" @@ -512,7 +558,6 @@ func (w *Worker) executeOne(proc *Processor) { } if carryErr != nil { ctxapi.ReleaseFrameContext(upgradeFrame) - proc.Process.Close() if !proc.casState(StateRunning, StateComplete) { return } @@ -520,9 +565,11 @@ func (w *Worker) executeOne(proc *Processor) { w.scheduler.complete(proc, nil, fmt.Errorf("upgrade: carry process state failed: %w", carryErr)) return } + // The replacement may retain frame-owned resources even if Init fails. + // Completion closes it before releasing its frame, exactly once. + proc.releaseIncarnation() + proc.ctx = upgradeCtx if err := newProc.Init(upgradeCtx, method, req.Input); err != nil { - ctxapi.ReleaseFrameContext(upgradeFrame) - proc.Process.Close() if !proc.casState(StateRunning, StateComplete) { return } @@ -530,8 +577,9 @@ func (w *Worker) executeOne(proc *Processor) { w.scheduler.complete(proc, nil, fmt.Errorf("upgrade: init failed: %w", err)) return } - proc.releaseIncarnation() - proc.ctx = upgradeCtx + if counter, ok := newProc.(process.StepAccounted); ok { + counter.ResumeStepCount(stepsUsed) + } // Re-publish the out-of-band snapshot so future invalidations classify // the process by its (possibly new) upgraded source. The queue // generation is unchanged by upgrade, so in-flight deliveries stay valid. @@ -549,7 +597,9 @@ func (w *Worker) executeOne(proc *Processor) { // dispatchYields sends all yields to handlers. // Processor state is StateRunning during this call. // CompleteYield sets wakeup flag instead of re-queueing while Running. -func (w *Worker) dispatchYields(ctx context.Context, proc *Processor, yields []process.Yield) { +// A preempted process stays runnable and is requeued; otherwise it blocks +// until a completion wakes it. +func (w *Worker) dispatchYields(ctx context.Context, proc *Processor, yields []process.Yield, preempted bool) { for _, y := range yields { handler := w.scheduler.getHandler(y.Cmd) if handler == nil { @@ -560,7 +610,7 @@ func (w *Worker) dispatchYields(ctx context.Context, proc *Processor, yields []p }) continue } - if err := handler.Handle(ctx, y.Cmd, y.Tag, proc); err != nil { + if err := handler.Handle(ctx, y.Cmd, y.Tag, proc.completer); err != nil { proc.queue.PushDirect(process.Event{ Type: process.EventYieldComplete, Tag: y.Tag, @@ -569,6 +619,13 @@ func (w *Worker) dispatchYields(ctx context.Context, proc *Processor, yields []p } } + if preempted { + if proc.casState(StateRunning, StateReady) { + w.requeuePreempted(proc) + } + return + } + // Atomically transition to final state. // If wakeup was set by CompleteYield, state becomes Ready - re-queue. // Otherwise, state becomes Blocked.