From aa47350a376ce3d993bebc069e862d2362bab8c5 Mon Sep 17 00:00:00 2001 From: David Porter Date: Wed, 11 Nov 2026 11:54:39 -0800 Subject: [PATCH 01/22] schedulertest: add scheduler lifecycle driver and property checks Squashed from https://github.com/temporalio/temporal/pull/13042 Source head: 33ffb9e001299e3221e107ce781f832b4a616a93 Original commits: 40c3ec56ee8edf6344d840c21167296ce822c652 chasm/chasmtest: expose node, backend and transaction close for test drivers de4543e53bd2b7646f7bfa80ac53d33ecea15fae schedulertest: scheduler lifecycle driver harness 216ee9f56f7dc1a0678a3f60b28542f1696d9cdb schedulertest: fix gofmt and importas lint violations 19b8e23de474da4df9fb4db398082517053c6c22 schedulertest: scheduler invariants and deterministic tables 40f1bfacdbb0783287853c19393a0c9a4b438be3 schedulertest: add buffer-bound and action-budget invariants fa8215d3dd3f7c86de2bb531fa8e0d173e638265 schedulertest: rapid property test for scheduler invariants 6984f62b5e1254591c420b484f417d057434e241 schedulertest: add default case to property-test op switch dc098f164a92b64d93f323c9bb41508ca96feaa6 schedulertest: port driver to current chasmtest engine 74c592cfcf81102238f2aa30c913625f4988dd57 scheduler: stamp InvokerExecuteTask start time from an injectable clock a8013369f2b7bf7b5e818926e1041b267b4113bd schedulertest: add a start ledger invariant 6495d8ae6fa86602c6deaf02337510db51441463 schedulertest: property test lagging side effects and schedules that end 33ffb9e001299e3221e107ce781f832b4a616a93 schedulertest: apply go fix modernization --- chasm/chasmtest/test_engine.go | 33 + chasm/lib/scheduler/invoker_tasks.go | 12 +- chasm/lib/scheduler/schedulertest/driver.go | 832 ++++++++++++++++++ .../schedulertest/driver_smoke_test.go | 72 ++ .../lib/scheduler/schedulertest/invariants.go | 172 ++++ .../schedulertest/invariants_test.go | 141 +++ .../scheduler/schedulertest/ledger_test.go | 206 +++++ chasm/lib/scheduler/schedulertest/library.go | 174 ++++ .../scheduler/schedulertest/property_test.go | 155 ++++ go.mod | 1 + go.sum | 2 + 11 files changed, 1799 insertions(+), 1 deletion(-) create mode 100644 chasm/lib/scheduler/schedulertest/driver.go create mode 100644 chasm/lib/scheduler/schedulertest/driver_smoke_test.go create mode 100644 chasm/lib/scheduler/schedulertest/invariants.go create mode 100644 chasm/lib/scheduler/schedulertest/invariants_test.go create mode 100644 chasm/lib/scheduler/schedulertest/ledger_test.go create mode 100644 chasm/lib/scheduler/schedulertest/library.go create mode 100644 chasm/lib/scheduler/schedulertest/property_test.go diff --git a/chasm/chasmtest/test_engine.go b/chasm/chasmtest/test_engine.go index 4724d374c0f..8b33b40c3cf 100644 --- a/chasm/chasmtest/test_engine.go +++ b/chasm/chasmtest/test_engine.go @@ -691,6 +691,39 @@ func (x *execution) recordRequestID(requestID string) { x.requestIDs[requestID] = struct{}{} } +// NodeForRef returns the CHASM tree node backing the execution identified by ref. +// It is intended for test drivers that dispatch tasks at the node level +// ([chasm.Node.ExecuteSideEffectTask]), which the [Engine] API does not expose. +func (e *Engine) NodeForRef(ref chasm.ComponentRef) (*chasm.Node, error) { + exec, err := e.executionForRef(ref) + if err != nil { + return nil, err + } + return exec.node, nil +} + +// BackendForRef returns the [chasm.MockNodeBackend] backing the execution +// identified by ref, e.g. to read the earliest pending pure task time via +// [chasm.MockNodeBackend.LastDeletePureTaskCall]. +func (e *Engine) BackendForRef(ref chasm.ComponentRef) (*chasm.MockNodeBackend, error) { + exec, err := e.executionForRef(ref) + if err != nil { + return nil, err + } + return exec.backend, nil +} + +// CloseTransaction closes any open transaction on the execution identified by +// ref, commits its transition count, and runs registered invariant checks. +// Pair it with node-level dispatch done through [Engine.NodeForRef]. +func (e *Engine) CloseTransaction(ctx context.Context, ref chasm.ComponentRef) error { + exec, err := e.executionForRef(ref) + if err != nil { + return err + } + return e.closeTransaction(ctx, exec) +} + // executionForRef looks up an execution by the ref's RunID when present, or falls back // to the current run for the business ID when RunID is empty. func (e *Engine) executionForRef(ref chasm.ComponentRef) (*execution, error) { diff --git a/chasm/lib/scheduler/invoker_tasks.go b/chasm/lib/scheduler/invoker_tasks.go index bb9234e77b2..d8853860d17 100644 --- a/chasm/lib/scheduler/invoker_tasks.go +++ b/chasm/lib/scheduler/invoker_tasks.go @@ -18,6 +18,7 @@ import ( "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" "go.temporal.io/server/chasm/lib/scheduler/internal" "go.temporal.io/server/common" + "go.temporal.io/server/common/clock" "go.temporal.io/server/common/log" "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/metrics" @@ -44,6 +45,9 @@ type ( // the request makes it through metering's interceptor. Because we don't change for // terminate/cancels, we can go directly to history for other service calls. FrontendClient workflowservice.WorkflowServiceClient + + // TimeSource stamps StartTime on started workflows. Defaults to real time. + TimeSource clock.TimeSource `optional:"true"` } InvokerExecuteTaskHandler struct { @@ -53,6 +57,7 @@ type ( baseLogger log.Logger historyClient resource.HistoryClient frontendClient workflowservice.WorkflowServiceClient + timeSource clock.TimeSource } InvokerProcessBufferTaskHandler struct { @@ -108,12 +113,17 @@ const ( ) func NewInvokerExecuteTaskHandler(opts InvokerTaskHandlerOptions) *InvokerExecuteTaskHandler { + timeSource := opts.TimeSource + if timeSource == nil { + timeSource = clock.NewRealTimeSource() + } return &InvokerExecuteTaskHandler{ config: opts.Config, metricsHandler: opts.MetricsHandler, baseLogger: opts.BaseLogger, historyClient: opts.HistoryClient, frontendClient: opts.FrontendClient, + timeSource: timeSource, } } @@ -741,7 +751,7 @@ func (h *InvokerExecuteTaskHandler) startWorkflow( if err != nil { return err } - actualStartTime := time.Now() + actualStartTime := h.timeSource.Now() // Reconstruct virtual time from wall time. actualStartTime = actualStartTime.Add( timeSkippingStatePropagation.GetInitialSkippedDuration().AsDuration(), diff --git a/chasm/lib/scheduler/schedulertest/driver.go b/chasm/lib/scheduler/schedulertest/driver.go new file mode 100644 index 00000000000..93e8b94cb89 --- /dev/null +++ b/chasm/lib/scheduler/schedulertest/driver.go @@ -0,0 +1,832 @@ +package schedulertest + +import ( + "context" + "errors" + "fmt" + "maps" + "sync" + "testing" + "time" + + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + schedulepb "go.temporal.io/api/schedule/v1" + "go.temporal.io/api/serviceerror" + workflowpb "go.temporal.io/api/workflow/v1" + "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/server/api/historyservice/v1" + "go.temporal.io/server/api/historyservicemock/v1" + persistencespb "go.temporal.io/server/api/persistence/v1" + schedulespb "go.temporal.io/server/api/schedule/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/chasmtest" + "go.temporal.io/server/chasm/lib/scheduler" + schedulerpb "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" + "go.temporal.io/server/common/clock" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/metrics/metricstest" + "go.temporal.io/server/common/testing/mockapi/workflowservicemock/v1" + "go.temporal.io/server/common/testing/testlogger" + "go.temporal.io/server/service/history/tasks" + "go.uber.org/mock/gomock" + "google.golang.org/grpc" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// maxKeyFireTime is the sentinel returned by MockNodeBackend.LastDeletePureTaskCall +// when the tree has no pending pure task. It matches the value passed by +// Node.closeTransactionGeneratePhysicalPureTask in that case. +var maxKeyFireTime = tasks.MaximumKey.FireTime + +// Driver builds a real CHASM scheduler on top of a [chasmtest.Engine] and steps +// it forward through every task it schedules for itself, advancing a controllable +// clock. After each settled transition it invokes AfterStep (if set), which is +// where invariant assertions are plugged in. +// +// A "step" fires every pure task due at the current reference time (via +// [chasm.Node.EachPureTask]) and every side-effect task whose visibility time has +// arrived (via [chasm.Node.ExecuteSideEffectTask]), then advances the clock to the +// next scheduled task. A healthy interval schedule never truly quiesces (it always +// re-arms a generator task), so callers bound the run with maxSteps; quiescence is +// reached only for paused/exhausted/closed schedules. +type Driver struct { + T *testing.T + Engine *chasmtest.Engine + Registry *chasm.Registry + TimeSource *clock.EventTimeSource + Logger log.Logger + + Frontend *workflowservicemock.MockWorkflowServiceClient + History *historyservicemock.MockHistoryServiceClient + + // Config is the scheduler config the library was wired with. Snapshot reads + // Tweakables (MaxBufferSize, MaxActionsPerExecution) from it so the + // buffer-bound and action-budget invariants can be evaluated. + Config *scheduler.Config + + ref chasm.ComponentRef + execKey chasm.ExecutionKey + + // dispatchedSideEffects tracks physical side-effect tasks already executed, + // keyed by pointer identity, since MockNodeBackend accumulates tasks append-only. + dispatchedSideEffects map[*tasks.ChasmTask]struct{} + + // actionRPCs points at the running count of action RPCs (StartWorkflowExecution + // + TerminateWorkflowExecution + RequestCancelWorkflowExecution) made through + // the mock clients. It is allocated before the clients so the stub closures can + // increment it. maxActionsObserved is the high-water mark of action RPCs made by + // any single side-effect task dispatch (one InvokerExecuteTask), which backs the + // action-budget invariant. + actionRPCs *int + maxActionsObserved int + + // startRequests records the RequestId of every StartWorkflowExecution RPC, and + // metricsCapture the scheduler's metrics; together they back the start ledger. + startRequests map[string]struct{} + metricsCapture *metricstest.Capture + // rpcMu guards actionRPCs and startRequests: InvokerExecuteTask issues + // starts from concurrent goroutines. + rpcMu *sync.Mutex + + // maxSideEffectLag caps how far StepPure may move the clock past the oldest + // undispatched side-effect task. Zero means unbounded. + maxSideEffectLag time.Duration + + // maxRunDuration auto-completes a started workflow before the clock moves + // past its StartTime plus this. Zero means workflows only complete through + // CompleteWorkflow. + maxRunDuration time.Duration + + // AfterStep, if set, is called after every step's CloseTransaction with the + // driver, so invariant checks can observe each settled state. + AfterStep func(d *Driver) +} + +type driverConfig struct { + schedule *schedulepb.Schedule + startTime time.Time + afterStep func(d *Driver) + config *scheduler.Config + maxLag time.Duration + maxRun time.Duration +} + +// DriverOption configures a Driver at construction. +type DriverOption func(*driverConfig) + +// WithSchedule overrides the schedule definition (default: DefaultSchedule). +func WithSchedule(s *schedulepb.Schedule) DriverOption { + return func(c *driverConfig) { c.schedule = s } +} + +// WithStartTime sets the wall-clock instant the engine's clock starts at. +func WithStartTime(t time.Time) DriverOption { + return func(c *driverConfig) { c.startTime = t } +} + +// WithAfterStep installs a hook called after every step. +func WithAfterStep(fn func(d *Driver)) DriverOption { + return func(c *driverConfig) { c.afterStep = fn } +} + +// WithConfig overrides the scheduler config (default: DefaultConfig). Use it to +// set small Tweakables — e.g. MaxBufferSize or MaxActionsPerExecution — so the +// buffer-bound and action-budget invariants are meaningfully exercised. +func WithConfig(cfg *scheduler.Config) DriverOption { + return func(c *driverConfig) { c.config = cfg } +} + +// WithMaxSideEffectLag bounds how long StepPure may leave a side-effect task +// (e.g. InvokerExecuteTask) undispatched while moving the clock forward. +func WithMaxSideEffectLag(lag time.Duration) DriverOption { + return func(c *driverConfig) { c.maxLag = lag } +} + +// WithMaxRunDuration bounds how long a started workflow runs: the driver +// completes it before moving the clock past StartTime plus d. +func WithMaxRunDuration(d time.Duration) DriverOption { + return func(c *driverConfig) { c.maxRun = d } +} + +// NewDriver constructs a Driver and starts a scheduler execution via the +// production CreateScheduler path. startTime defaults to a fixed instant for +// determinism. +func NewDriver(t *testing.T, opts ...DriverOption) *Driver { + t.Helper() + d, cfg := newDriverScaffold(t, opts...) + + req := &schedulerpb.CreateScheduleRequest{ + NamespaceId: NamespaceID, + FrontendRequest: &workflowservice.CreateScheduleRequest{ + Namespace: Namespace, + ScheduleId: ScheduleID, + Schedule: cfg.schedule, + RequestId: "create-req", + }, + } + d.startExecution(func(mctx chasm.MutableContext) (*scheduler.Scheduler, error) { + return scheduler.CreateScheduler(mctx, req) + }) + return d +} + +// NewDriverFromMigration constructs a Driver and starts a scheduler execution +// from a migrated V1 state via the production CreateSchedulerFromMigration path. +// This is the V1->V2 import side of a migration round trip. +func NewDriverFromMigration(t *testing.T, req *schedulerpb.CreateFromMigrationStateRequest, opts ...DriverOption) *Driver { + t.Helper() + d, _ := newDriverScaffold(t, opts...) + d.startExecution(func(mctx chasm.MutableContext) (*scheduler.Scheduler, error) { + return scheduler.CreateSchedulerFromMigration(mctx, req) + }) + return d +} + +// newDriverScaffold builds the engine, registry, clients, and time source, but +// does not start an execution. +func newDriverScaffold(t *testing.T, opts ...DriverOption) (*Driver, *driverConfig) { + t.Helper() + + cfg := &driverConfig{ + schedule: DefaultSchedule(), + startTime: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC), + config: DefaultConfig(), + } + for _, opt := range opts { + opt(cfg) + } + + ctrl := gomock.NewController(t) + logger := testlogger.NewTestLogger(t, testlogger.FailOnExpectedErrorOnly) + + ts := clock.NewEventTimeSource() + ts.Update(cfg.startTime) + + frontend := workflowservicemock.NewMockWorkflowServiceClient(ctrl) + history := historyservicemock.NewMockHistoryServiceClient(ctrl) + + // actionRPCs counts every action RPC (start/terminate/cancel) the invoker + // makes through the mock clients, so Step can measure how many actions a single + // ExecuteTask takes (the action-budget invariant). + actionRPCs := new(int) + startRequests := make(map[string]struct{}) + rpcMu := &sync.Mutex{} + countAction := func() { + rpcMu.Lock() + defer rpcMu.Unlock() + *actionRPCs++ + } + + // Default happy-path stubs; tests can layer more specific expectations. + frontend.EXPECT().StartWorkflowExecution(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, req *workflowservice.StartWorkflowExecutionRequest, _ ...grpc.CallOption) (*workflowservice.StartWorkflowExecutionResponse, error) { + countAction() + rpcMu.Lock() + startRequests[req.GetRequestId()] = struct{}{} + rpcMu.Unlock() + return &workflowservice.StartWorkflowExecutionResponse{RunId: "run-id"}, nil + }).AnyTimes() + history.EXPECT().TerminateWorkflowExecution(gomock.Any(), gomock.Any()). + DoAndReturn(func(context.Context, *historyservice.TerminateWorkflowExecutionRequest, ...grpc.CallOption) (*historyservice.TerminateWorkflowExecutionResponse, error) { + countAction() + return &historyservice.TerminateWorkflowExecutionResponse{}, nil + }).AnyTimes() + history.EXPECT().RequestCancelWorkflowExecution(gomock.Any(), gomock.Any()). + DoAndReturn(func(context.Context, *historyservice.RequestCancelWorkflowExecutionRequest, ...grpc.CallOption) (*historyservice.RequestCancelWorkflowExecutionResponse, error) { + countAction() + return &historyservice.RequestCancelWorkflowExecutionResponse{}, nil + }).AnyTimes() + // The callbacks task (used for migrated running workflows) describes each + // running workflow; report it completed so watchers resolve deterministically. + history.EXPECT().DescribeWorkflowExecution(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, _ *historyservice.DescribeWorkflowExecutionRequest, _ ...grpc.CallOption) (*historyservice.DescribeWorkflowExecutionResponse, error) { + return &historyservice.DescribeWorkflowExecutionResponse{ + WorkflowExecutionInfo: &workflowpb.WorkflowExecutionInfo{ + Status: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, + CloseTime: timestamppb.New(ts.Now()), + }, + }, nil + }).AnyTimes() + + specProcessor := NewRealSpecProcessor(ctrl, logger) + + registry := chasm.NewRegistry(logger) + if err := registry.Register(&chasm.CoreLibrary{}); err != nil { + t.Fatalf("failed to register core library: %v", err) + } + metricsHandler := metricstest.NewCaptureHandler() + capture := metricsHandler.StartCapture() + if err := registry.Register(NewTestLibrary(cfg.config, logger, metricsHandler, ts, specProcessor, frontend, history)); err != nil { + t.Fatalf("failed to register scheduler library: %v", err) + } + + engine := chasmtest.NewEngine(t, registry, chasmtest.WithTimeSource(ts)) + + d := &Driver{ + T: t, + Engine: engine, + Registry: registry, + TimeSource: ts, + Logger: logger, + Frontend: frontend, + History: history, + Config: cfg.config, + dispatchedSideEffects: make(map[*tasks.ChasmTask]struct{}), + actionRPCs: actionRPCs, + startRequests: startRequests, + rpcMu: rpcMu, + metricsCapture: capture, + maxSideEffectLag: cfg.maxLag, + maxRunDuration: cfg.maxRun, + AfterStep: cfg.afterStep, + } + return d, cfg +} + +// startExecution starts the scheduler execution using the given start function. +func (d *Driver) startExecution(startFn func(chasm.MutableContext) (*scheduler.Scheduler, error)) { + d.T.Helper() + + d.execKey = chasm.ExecutionKey{NamespaceID: NamespaceID, BusinessID: ScheduleID} + engineCtx := chasm.NewEngineContext(context.Background(), d.Engine) + + _, err := chasm.StartExecution(engineCtx, d.execKey, func(mctx chasm.MutableContext, _ *struct{}) (*scheduler.Scheduler, error) { + return startFn(mctx) + }, (*struct{})(nil), chasm.WithRequestID("create-req")) + if err != nil { + d.T.Fatalf("failed to start scheduler execution: %v", err) + } + + d.ref = chasm.NewComponentRef[*scheduler.Scheduler](d.execKey) +} + +// Now returns the engine's current clock time. +func (d *Driver) Now() time.Time { + return d.TimeSource.Now() +} + +// ReadScheduler reads the scheduler root component and invokes fn with it. +func (d *Driver) ReadScheduler(fn func(s *scheduler.Scheduler, ctx chasm.Context)) { + d.T.Helper() + engineCtx := chasm.NewEngineContext(context.Background(), d.Engine) + err := d.Engine.ReadComponent(engineCtx, d.ref, func(ctx chasm.Context, c chasm.Component) error { + s, ok := c.(*scheduler.Scheduler) + if !ok { + d.T.Fatalf("root component is %T, want *scheduler.Scheduler", c) + } + fn(s, ctx) + return nil + }) + if err != nil { + d.T.Fatalf("ReadScheduler failed: %v", err) + } +} + +// node returns the CHASM tree node backing the scheduler execution. +func (d *Driver) node() *chasm.Node { + d.T.Helper() + node, err := d.Engine.NodeForRef(d.ref) + if err != nil { + d.T.Fatalf("NodeForRef failed: %v", err) + } + return node +} + +func (d *Driver) backend() *chasm.MockNodeBackend { + d.T.Helper() + backend, err := d.Engine.BackendForRef(d.ref) + if err != nil { + d.T.Fatalf("BackendForRef failed: %v", err) + } + return backend +} + +// nextPureFireTime returns the earliest pending pure task time, if any. It is +// recorded by closeTransactionGeneratePhysicalPureTask via +// DeleteCHASMPureTasks(earliestPureTaskTime), or MaximumKey.FireTime if none. +func (d *Driver) nextPureFireTime() (time.Time, bool) { + pureTime := d.backend().LastDeletePureTaskCall() + if pureTime.IsZero() || pureTime.Equal(maxKeyFireTime) { + return time.Time{}, false + } + return pureTime, true +} + +// oldestSideEffectTime returns the earliest visibility time among undispatched +// side-effect tasks, if any. +func (d *Driver) oldestSideEffectTime() (time.Time, bool) { + var oldest time.Time + have := false + for _, t := range d.pendingSideEffectTasks() { + if vt := t.GetVisibilityTime(); !have || vt.Before(oldest) { + oldest, have = vt, true + } + } + return oldest, have +} + +// nextFireTime returns the earliest time at which any pending task (pure or +// side-effect) is due, and whether any task is pending at all. +func (d *Driver) nextFireTime() (time.Time, bool) { + pure, havePure := d.nextPureFireTime() + side, haveSide := d.oldestSideEffectTime() + switch { + case havePure && haveSide: + if side.Before(pure) { + return side, true + } + return pure, true + case havePure: + return pure, true + default: + return side, haveSide + } +} + +// pendingSideEffectTasks returns physical side-effect tasks that have not yet +// been dispatched by this driver. +func (d *Driver) pendingSideEffectTasks() []*tasks.ChasmTask { + backend := d.backend() + var out []*tasks.ChasmTask + for category, categoryTasks := range backend.TasksByCategory { + // Visibility tasks are framework-managed (indexing search attributes) and + // processed by a separate queue, not the CHASM side-effect executor; their + // handler panics if invoked directly. Skip them. + if category.ID() == tasks.CategoryVisibility.ID() { + continue + } + for _, t := range categoryTasks { + ct, ok := t.(*tasks.ChasmTask) + if !ok { + continue + } + if _, done := d.dispatchedSideEffects[ct]; done { + continue + } + out = append(out, ct) + } + } + return out +} + +// Step advances the clock to the next scheduled task and fires all tasks due at +// that time: pure tasks first, then every due side-effect task. It returns false +// if nothing was pending (the schedule is quiescent). +func (d *Driver) Step() bool { + d.T.Helper() + + fireTime, have := d.nextFireTime() + if !have { + return false + } + d.advanceTo(fireTime) + d.firePureTasks() + for d.StepSideEffect() { + } + return true +} + +// StepPure advances the clock to the next pending pure task and fires every pure +// task due then, leaving side-effect tasks undispatched. This models +// InvokerExecuteTask lagging behind the scheduler's timers. It returns false if +// no pure task is pending, or if firing it would move the clock more than +// maxSideEffectLag past the oldest undispatched side-effect task. +func (d *Driver) StepPure() bool { + d.T.Helper() + + fireTime, have := d.nextPureFireTime() + if !have { + return false + } + if oldest, ok := d.oldestSideEffectTime(); ok && d.maxSideEffectLag > 0 && + fireTime.After(oldest.Add(d.maxSideEffectLag)) { + return false + } + d.advanceTo(fireTime) + d.firePureTasks() + return true +} + +// StepSideEffect dispatches the oldest side-effect task that is due at the +// current time, without moving the clock. It returns false if none is due. +func (d *Driver) StepSideEffect() bool { + d.T.Helper() + + now := d.Now() + var next *tasks.ChasmTask + for _, ct := range d.pendingSideEffectTasks() { + if ct.GetVisibilityTime().After(now) { + continue + } + if next == nil || ct.GetVisibilityTime().Before(next.GetVisibilityTime()) { + next = ct + } + } + if next == nil { + return false + } + + d.dispatchedSideEffects[next] = struct{}{} + actionsBefore := d.actionCount() + execErr := d.node().ExecuteSideEffectTask( + chasm.NewEngineContext(context.Background(), d.Engine), + d.execKey, + next, + func(chasm.NodeBackend, chasm.Context, chasm.Component) error { return nil }, + ) + if execErr != nil { + // A NotFound means the logical task was superseded/removed before + // dispatch (the backend accumulates physical tasks append-only, so a + // stale one can linger). Production drops these; so do we. + if _, ok := errors.AsType[*serviceerror.NotFound](execErr); !ok { + d.T.Fatalf("ExecuteSideEffectTask failed: %v", execErr) + } + } + // Track the most actions any single side-effect task (i.e. one + // InvokerExecuteTask) took, for the action-budget invariant. + if taken := d.actionCount() - actionsBefore; taken > d.maxActionsObserved { + d.maxActionsObserved = taken + } + if err := d.Engine.CloseTransaction(context.Background(), d.ref); err != nil { + d.T.Fatalf("CloseTransaction after side-effect task failed: %v", err) + } + if d.AfterStep != nil { + d.AfterStep(d) + } + return true +} + +// advanceTo moves the clock to t, never backward, first completing any +// workflow that would exceed maxRunDuration on the way. +func (d *Driver) advanceTo(t time.Time) { + for d.maxRunDuration > 0 { + startTime, ok := d.oldestRunningStartTime() + if !ok { + break + } + due := startTime.Add(d.maxRunDuration) + if due.After(t) { + break + } + if due.After(d.Now()) { + d.TimeSource.Update(due) + } + d.CompleteWorkflow() + // The completion may ready a deferred start; don't skip its execute task. + for d.StepSideEffect() { + } + } + if t.After(d.Now()) { + d.TimeSource.Update(t) + } +} + +// oldestRunningStartTime returns the StartTime of the oldest started workflow +// that hasn't completed. +func (d *Driver) oldestRunningStartTime() (time.Time, bool) { + var oldest time.Time + have := false + d.ReadScheduler(func(s *scheduler.Scheduler, ctx chasm.Context) { + if s.Closed || s.IsSentinel() { + return + } + for _, start := range s.Invoker.Get(ctx).GetBufferedStarts() { + if start.GetRunId() == "" || start.GetCompleted() != nil { + continue + } + if st := start.GetStartTime().AsTime(); !have || st.Before(oldest) { + oldest, have = st, true + } + } + }) + return oldest, have +} + +// CompleteWorkflow delivers a successful completion callback for the oldest +// running workflow, through Scheduler.HandleNexusCompletion. It returns false +// if no workflow is running. +func (d *Driver) CompleteWorkflow() bool { + d.T.Helper() + completed := false + engineCtx := chasm.NewEngineContext(context.Background(), d.Engine) + _, err := d.Engine.UpdateComponent(engineCtx, d.ref, func(mctx chasm.MutableContext, c chasm.Component) error { + s, ok := c.(*scheduler.Scheduler) + if !ok { + return fmt.Errorf("root component is %T, want *scheduler.Scheduler", c) + } + if s.Closed || s.IsSentinel() { + return nil + } + var oldest *schedulespb.BufferedStart + for _, start := range s.Invoker.Get(mctx).GetBufferedStarts() { + if start.GetRunId() == "" || start.GetCompleted() != nil { + continue + } + if oldest == nil || start.GetStartTime().AsTime().Before(oldest.GetStartTime().AsTime()) { + oldest = start + } + } + if oldest == nil { + return nil + } + completed = true + return s.HandleNexusCompletion(mctx, &persistencespb.ChasmNexusCompletion{ + RequestId: oldest.GetRequestId(), + Outcome: &persistencespb.ChasmNexusCompletion_Success{Success: &commonpb.Payload{}}, + CloseTime: timestamppb.New(d.Now()), + }) + }) + if err != nil { + d.T.Fatalf("CompleteWorkflow failed: %v", err) + } + if completed && d.AfterStep != nil { + d.AfterStep(d) + } + return completed +} + +// firePureTasks fires every pure task due at the current time and commits. +func (d *Driver) firePureTasks() { + d.T.Helper() + now := d.Now() + pureCtx := chasm.NewEngineContext(context.Background(), d.Engine) + err := d.node().EachPureTask(now, func(executor chasm.NodePureTask, attrs chasm.TaskAttributes, taskInstance any) (bool, error) { + return executor.ExecutePureTask(pureCtx, attrs, taskInstance) + }) + if err != nil { + d.T.Fatalf("EachPureTask failed: %v", err) + } + if err := d.Engine.CloseTransaction(context.Background(), d.ref); err != nil { + d.T.Fatalf("CloseTransaction after pure tasks failed: %v", err) + } + if d.AfterStep != nil { + d.AfterStep(d) + } +} + +// RunToQuiescence steps until no task is pending or maxSteps is reached. It +// returns the number of steps taken and whether the schedule reached quiescence +// (no pending task). A normal running schedule never quiesces and will stop at +// maxSteps with quiescent == false. +func (d *Driver) RunToQuiescence(maxSteps int) (steps int, quiescent bool) { + d.T.Helper() + for steps = 0; steps < maxSteps; steps++ { + if !d.Step() { + return steps, true + } + } + return steps, false +} + +// HasPendingTask reports whether any task (pure or side-effect) is currently +// scheduled to fire. +func (d *Driver) HasPendingTask() bool { + _, have := d.nextFireTime() + return have +} + +// Patch applies a SchedulePatch through the production Scheduler.Patch path in a +// single transaction. It returns the scheduler's error (e.g. ErrClosed when the +// schedule has already closed) without failing the test, so callers driving +// random operations can tolerate rejected patches. On success the AfterStep hook +// runs so invariants are checked at the new settled state. +func (d *Driver) Patch(patch *schedulepb.SchedulePatch) error { + d.T.Helper() + engineCtx := chasm.NewEngineContext(context.Background(), d.Engine) + _, err := d.Engine.UpdateComponent(engineCtx, d.ref, func(mctx chasm.MutableContext, c chasm.Component) error { + s, ok := c.(*scheduler.Scheduler) + if !ok { + return fmt.Errorf("root component is %T, want *scheduler.Scheduler", c) + } + _, perr := s.Patch(mctx, &schedulerpb.PatchScheduleRequest{ + NamespaceId: NamespaceID, + FrontendRequest: &workflowservice.PatchScheduleRequest{ + Namespace: Namespace, + ScheduleId: ScheduleID, + Patch: patch, + }, + }) + return perr + }) + if err == nil && d.AfterStep != nil { + d.AfterStep(d) + } + return err +} + +// Pause pauses the schedule via Patch. +func (d *Driver) Pause() error { + return d.Patch(&schedulepb.SchedulePatch{Pause: "paused by driver"}) +} + +// Unpause unpauses the schedule via Patch. +func (d *Driver) Unpause() error { + return d.Patch(&schedulepb.SchedulePatch{Unpause: "unpaused by driver"}) +} + +// TriggerImmediately requests an immediate action via Patch. +func (d *Driver) TriggerImmediately() error { + return d.Patch(&schedulepb.SchedulePatch{ + TriggerImmediately: &schedulepb.TriggerImmediatelyRequest{}, + }) +} + +// MigrateToWorkflow initiates a V2->V1 migration: it pauses the schedule and +// schedules the SchedulerMigrateToWorkflowTask side-effect task. Step the driver +// afterward to fire the task (which calls the history client's +// StartWorkflowExecution with the exported StartScheduleArgs). +func (d *Driver) MigrateToWorkflow() error { + d.T.Helper() + engineCtx := chasm.NewEngineContext(context.Background(), d.Engine) + _, err := d.Engine.UpdateComponent(engineCtx, d.ref, func(mctx chasm.MutableContext, c chasm.Component) error { + s, ok := c.(*scheduler.Scheduler) + if !ok { + return fmt.Errorf("root component is %T, want *scheduler.Scheduler", c) + } + _, merr := s.MigrateToWorkflow(mctx, &schedulerpb.MigrateToWorkflowRequest{ + NamespaceId: NamespaceID, + ScheduleId: ScheduleID, + }) + return merr + }) + return err +} + +// Snapshot captures the observable scheduler state at a settled point (after a +// step's CloseTransaction). It is the input to invariant checks. +type Snapshot struct { + // Now is the engine clock at capture time. + Now time.Time + // HasPendingTask is true if any task is scheduled to fire. + HasPendingTask bool + + Closed bool + IsSentinel bool + Paused bool + // BackfillerCount is the number of live backfiller components. + BackfillerCount int + // IsHeldOpen mirrors Scheduler.isHeldOpen: the schedule must stay open + // regardless of having no work (paused or pending backfill, non-sentinel). + IsHeldOpen bool + // IdleCloseTime is the armed idle-close deadline, or zero if no idle task is armed. + IdleCloseTime time.Time + + // GeneratorLPT / InvokerLPT are the generator and invoker high-water marks. + GeneratorLPT time.Time + InvokerLPT time.Time + + // BufferedStartsCount is len(invoker.GetBufferedStarts()) — the total buffered + // starts (pending + retained). MaxBufferSize is the configured cap; the + // buffer-bound invariant requires BufferedStartsCount <= MaxBufferSize. + BufferedStartsCount int + MaxBufferSize int + + // MaxActionsObserved is the high-water mark of action RPCs (start + terminate + + // cancel) taken by any single InvokerExecuteTask so far. MaxActionsPerExecution + // is the configured per-execution cap; the action-budget invariant requires + // MaxActionsObserved <= MaxActionsPerExecution. + MaxActionsObserved int + MaxActionsPerExecution int + + // Start ledger inputs. PendingStarts are buffered starts with no RunId yet. + // StartRequested holds every RequestId sent to StartWorkflowExecution. + // OverlapSkipped and DroppedStarts count the starts the scheduler discarded + // on purpose (Info.OverlapSkipped, and the ScheduleBufferedStartDropped + // metric for missed-catchup and paused/limited drops). + PendingStarts map[string]struct{} + StartRequested map[string]struct{} + OverlapSkipped int64 + DroppedStarts int64 + // Migrating is true once the schedule hands off to a V1 workflow, which + // takes its buffered starts with it. + Migrating bool + // DeferredStarts are pending starts waiting on a workflow completion + // (Attempt == -1); RunningWorkflows counts started, uncompleted workflows. + DeferredStarts int + RunningWorkflows int +} + +// Snapshot reads current scheduler state into a Snapshot. +func (d *Driver) Snapshot() Snapshot { + d.T.Helper() + tweakables := d.Config.Tweakables(Namespace) + snap := Snapshot{ + Now: d.Now(), + HasPendingTask: d.HasPendingTask(), + MaxBufferSize: tweakables.MaxBufferSize, + MaxActionsObserved: d.maxActionsObserved, + MaxActionsPerExecution: tweakables.MaxActionsPerExecution, + PendingStarts: make(map[string]struct{}), + StartRequested: d.startRequestsCopy(), + DroppedStarts: d.droppedStarts(), + } + d.ReadScheduler(func(s *scheduler.Scheduler, ctx chasm.Context) { + snap.Closed = s.Closed + snap.IsSentinel = s.IsSentinel() + snap.Paused = s.Schedule.GetState().GetPaused() + snap.BackfillerCount = len(s.Backfillers) + snap.OverlapSkipped = s.Info.GetOverlapSkipped() + snap.Migrating = s.GetWorkflowMigration() != nil + // Mirror Scheduler.isHeldOpen() from public state. + snap.IsHeldOpen = !snap.IsSentinel && (snap.Paused || snap.BackfillerCount > 0) + if s.IdleCloseTime != nil { + snap.IdleCloseTime = s.IdleCloseTime.AsTime() + } + // Sentinels have no generator/invoker sub-components. + if !snap.IsSentinel { + if g := s.Generator.Get(ctx); g != nil && g.LastProcessedTime != nil { + snap.GeneratorLPT = g.LastProcessedTime.AsTime() + } + if inv := s.Invoker.Get(ctx); inv != nil { + snap.BufferedStartsCount = len(inv.GetBufferedStarts()) + snap.addBufferedStarts(inv.GetBufferedStarts()) + if inv.LastProcessedTime != nil { + snap.InvokerLPT = inv.LastProcessedTime.AsTime() + } + } + } + }) + return snap +} + +// addBufferedStarts classifies buffered starts into pending (deferred or not) +// and running. +func (snap *Snapshot) addBufferedStarts(starts []*schedulespb.BufferedStart) { + for _, start := range starts { + switch { + case start.GetRunId() == "": + snap.PendingStarts[start.GetRequestId()] = struct{}{} + if start.GetAttempt() == -1 { + snap.DeferredStarts++ + } + case start.GetCompleted() == nil: + snap.RunningWorkflows++ + default: + } + } +} + +// droppedStarts sums the ScheduleBufferedStartDropped counter. +func (d *Driver) droppedStarts() int64 { + var total int64 + for _, rec := range d.metricsCapture.SnapshotMetric(metrics.ScheduleBufferedStartDropped.Name()) { + if v, ok := rec.Value.(int64); ok { + total += v + } + } + return total +} + +func (d *Driver) actionCount() int { + d.rpcMu.Lock() + defer d.rpcMu.Unlock() + return *d.actionRPCs +} + +func (d *Driver) startRequestsCopy() map[string]struct{} { + d.rpcMu.Lock() + defer d.rpcMu.Unlock() + return maps.Clone(d.startRequests) +} diff --git a/chasm/lib/scheduler/schedulertest/driver_smoke_test.go b/chasm/lib/scheduler/schedulertest/driver_smoke_test.go new file mode 100644 index 00000000000..4f80dfa219a --- /dev/null +++ b/chasm/lib/scheduler/schedulertest/driver_smoke_test.go @@ -0,0 +1,72 @@ +package schedulertest_test + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/scheduler" + "go.temporal.io/server/chasm/lib/scheduler/schedulertest" +) + +// TestDriver_RunsIntervalScheduleForward verifies the driver fires the +// scheduler's self-scheduled tasks and advances the clock across several +// interval ticks of a normal (running) schedule. +func TestDriver_RunsIntervalScheduleForward(t *testing.T) { + start := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) + d := schedulertest.NewDriver(t, schedulertest.WithStartTime(start)) + + steps, quiescent := d.RunToQuiescence(20) + + // A normal interval schedule re-arms a generator task forever, so it never + // quiesces; it should burn the whole step budget. + require.False(t, quiescent, "running interval schedule should not quiesce") + require.Equal(t, 20, steps) + + // The clock must have advanced as interval ticks fire. + require.True(t, d.Now().After(start.Add(schedulertest.DefaultInterval)), + "clock should advance past one interval, now=%s", d.Now()) + + d.ReadScheduler(func(s *scheduler.Scheduler, _ chasm.Context) { + require.False(t, s.Closed, "schedule should still be open") + }) +} + +// TestDriver_PausedScheduleKeepsTicking documents that a paused interval +// schedule is *held open* and keeps re-arming its generator task to advance the +// high-water mark — it neither quiesces nor closes — but it never buffers a +// start. +func TestDriver_PausedScheduleKeepsTicking(t *testing.T) { + sched := schedulertest.DefaultSchedule() + sched.State.Paused = true + + d := schedulertest.NewDriver(t, schedulertest.WithSchedule(sched)) + + _, quiescent := d.RunToQuiescence(10) + require.False(t, quiescent, "paused interval schedule keeps ticking, not quiescent") + + d.ReadScheduler(func(s *scheduler.Scheduler, _ chasm.Context) { + require.True(t, s.Schedule.State.Paused) + require.False(t, s.Closed, "paused schedule is held open, not closed") + }) +} + +// TestDriver_LimitedActionsExhaustedClosesAndQuiesces verifies that a schedule +// with no remaining actions arms an idle task instead of generating work, and +// after the idle task fires the schedule closes and the driver reaches +// quiescence. +func TestDriver_LimitedActionsExhaustedClosesAndQuiesces(t *testing.T) { + sched := schedulertest.DefaultSchedule() + sched.State.LimitedActions = true + sched.State.RemainingActions = 0 + + d := schedulertest.NewDriver(t, schedulertest.WithSchedule(sched)) + + _, quiescent := d.RunToQuiescence(10) + require.True(t, quiescent, "exhausted schedule should close and quiesce") + + d.ReadScheduler(func(s *scheduler.Scheduler, _ chasm.Context) { + require.True(t, s.Closed, "schedule should be closed after idle task fires") + }) +} diff --git a/chasm/lib/scheduler/schedulertest/invariants.go b/chasm/lib/scheduler/schedulertest/invariants.go new file mode 100644 index 00000000000..f65ed68f741 --- /dev/null +++ b/chasm/lib/scheduler/schedulertest/invariants.go @@ -0,0 +1,172 @@ +package schedulertest + +import ( + "fmt" + "maps" + "slices" +) + +// Violation describes a single broken invariant. +type Violation struct { + Name string + Message string +} + +func (v Violation) String() string { + return fmt.Sprintf("[%s] %s", v.Name, v.Message) +} + +// CheckInvariants evaluates the scheduler invariants against a settled state +// (cur). When prev is non-nil it also checks cross-step invariants (monotonicity, +// closed-is-terminal). It returns all violations found; an empty slice means the +// state is healthy. +// +// The invariants encode the legitimate-quiescence rules from Scheduler.isHeldOpen +// and Scheduler.getIdleExpiration: a non-closed schedule may have no pending task +// only when it is held open (paused or draining a backfill). Any other taskless, +// non-closed state is the "stuck" bug class. +func CheckInvariants(prev *Snapshot, cur Snapshot) []Violation { + var violations []Violation + add := func(name, format string, args ...any) { + violations = append(violations, Violation{Name: name, Message: fmt.Sprintf(format, args...)}) + } + + // (1) No-stuck (primary): a live schedule must always have a way to make + // progress. The only taskless, non-closed state allowed is "held open" + // (paused or pending backfill). When idle is armed there is a pending idle + // task, so HasPendingTask covers that case. + if !cur.Closed && !cur.HasPendingTask && !cur.IsHeldOpen { + add("no-stuck", + "scheduler has no pending task, is not closed, and is not held open "+ + "(paused=%v backfillers=%d) at now=%s — stuck", + cur.Paused, cur.BackfillerCount, cur.Now.Format("2006-01-02T15:04:05")) + } + + // (2) Idle correctness: a held-open schedule must never have an idle-close + // armed; idle close would race the customer's intent to resume/drain. + if cur.IsHeldOpen && !cur.IdleCloseTime.IsZero() { + add("idle-not-while-held-open", + "scheduler is held open (paused=%v backfillers=%d) but has idle close armed at %s", + cur.Paused, cur.BackfillerCount, cur.IdleCloseTime.Format("2006-01-02T15:04:05")) + } + + // (2a) Buffer bound: the total buffered starts must never exceed MaxBufferSize. + // The generator sizes each fill against this cap (remaining = MaxBufferSize - + // len(buffered)); exceeding it means the limit was not enforced and CHASM state + // can grow without bound (oversized state, persistence failures, OOM). + if cur.MaxBufferSize > 0 && cur.BufferedStartsCount > cur.MaxBufferSize { + add("buffer-bound", + "buffered starts = %d, exceeding MaxBufferSize=%d", + cur.BufferedStartsCount, cur.MaxBufferSize) + } + + // (2b) Action budget: a single InvokerExecuteTask must not take more actions + // (start + terminate + cancel RPCs) than MaxActionsPerExecution. Exceeding it + // means the per-execution rate protection was bypassed — e.g. the action budget + // reset across the terminate/cancel/start phases, letting one execution take up + // to a multiple of the configured maximum. + if cur.MaxActionsPerExecution > 0 && cur.MaxActionsObserved > cur.MaxActionsPerExecution { + add("action-budget", + "a single ExecuteTask took %d actions, exceeding MaxActionsPerExecution=%d", + cur.MaxActionsObserved, cur.MaxActionsPerExecution) + } + + violations = append(violations, checkLedger(prev, cur)...) + + if prev != nil { + // (3) Closed is terminal: a closed schedule must never reopen. + if prev.Closed && !cur.Closed { + add("closed-terminal", "scheduler reopened after being closed") + } + + // (4) High-water-mark monotonicity: generator/invoker HWMs never regress. + if cur.GeneratorLPT.Before(prev.GeneratorLPT) { + add("hwm-monotonic-generator", + "generator high-water mark went backward: %s -> %s", + prev.GeneratorLPT.Format("2006-01-02T15:04:05"), + cur.GeneratorLPT.Format("2006-01-02T15:04:05")) + } + if cur.InvokerLPT.Before(prev.InvokerLPT) { + add("hwm-monotonic-invoker", + "invoker high-water mark went backward: %s -> %s", + prev.InvokerLPT.Format("2006-01-02T15:04:05"), + cur.InvokerLPT.Format("2006-01-02T15:04:05")) + } + } + + return violations +} + +// fataler is the subset of testing.TB the invariant assertion helper needs. +type fataler interface { + Helper() + Fatalf(format string, args ...any) +} + +// CheckInvariantsHook returns an AfterStep hook that captures a snapshot after +// every step and fails the test on the first invariant violation. It threads the +// previous snapshot so cross-step invariants are checked. This is the normal way +// invariant checking is wired into a Driver: +// +// d := NewDriver(t) +// d.AfterStep = CheckInvariantsHook(t) +func CheckInvariantsHook(t fataler) func(d *Driver) { + var prev *Snapshot + return func(d *Driver) { + t.Helper() + cur := d.Snapshot() + if violations := CheckInvariants(prev, cur); len(violations) > 0 { + t.Fatalf("scheduler invariant violation at step (now=%s): %s", + cur.Now.Format("2006-01-02T15:04:05"), violations[0]) + } + snap := cur + prev = &snap + } +} + +// checkLedger checks that every start the scheduler buffered is either sent to +// StartWorkflowExecution or discarded on purpose, and never lost to a close. +func checkLedger(prev *Snapshot, cur Snapshot) []Violation { + var violations []Violation + add := func(name, format string, args ...any) { + violations = append(violations, Violation{Name: name, Message: fmt.Sprintf(format, args...)}) + } + + if cur.Migrating { + // A migration hands the buffer to the V1 workflow. + return nil + } + + // Terminal half: a schedule never closes while a start it buffered is + // still waiting for InvokerExecuteTask (temporal#13016). + if cur.Closed && len(cur.PendingStarts) > 0 { + add("ledger-lost-start", + "scheduler closed with %d buffered start(s) never sent to StartWorkflowExecution: %v", + len(cur.PendingStarts), slices.Sorted(maps.Keys(cur.PendingStarts))) + } + if prev == nil { + return violations + } + + // Conservation half: every start that leaves the buffer without a RunId was + // either sent to StartWorkflowExecution or discarded on purpose (overlap + // skip, missed catchup, paused/limited drop). Starts that are buffered and + // discarded within one transaction never appear pending, so the discard + // counters can only exceed the disappearances, never trail. + var silent []string + for id := range prev.PendingStarts { + _, still := cur.PendingStarts[id] + _, sent := cur.StartRequested[id] + if !still && !sent { + silent = append(silent, id) + } + } + discarded := (cur.OverlapSkipped - prev.OverlapSkipped) + (cur.DroppedStarts - prev.DroppedStarts) + if int64(len(silent)) > discarded { + slices.Sort(silent) + add("ledger-silent-drop", + "%d buffered start(s) left the buffer without a start RPC, but only %d discard(s) were recorded: %v", + len(silent), discarded, silent) + } + return violations +} diff --git a/chasm/lib/scheduler/schedulertest/invariants_test.go b/chasm/lib/scheduler/schedulertest/invariants_test.go new file mode 100644 index 00000000000..8997218eb7c --- /dev/null +++ b/chasm/lib/scheduler/schedulertest/invariants_test.go @@ -0,0 +1,141 @@ +package schedulertest_test + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + schedulepb "go.temporal.io/api/schedule/v1" + "go.temporal.io/server/chasm/lib/scheduler/schedulertest" + "google.golang.org/protobuf/types/known/durationpb" +) + +// TestInvariants_HoldAcrossScenarios runs a variety of schedules forward with +// the invariant hook wired in and asserts no invariant fires. Each scenario +// exercises a different legitimate-quiescence path (running, paused, exhausted, +// end-time passed). +func TestInvariants_HoldAcrossScenarios(t *testing.T) { + cases := []struct { + name string + schedule func() *schedulepb.Schedule + maxSteps int + }{ + { + name: "running interval", + schedule: schedulertest.DefaultSchedule, + maxSteps: 30, + }, + { + name: "paused interval (held open, keeps ticking)", + schedule: func() *schedulepb.Schedule { + s := schedulertest.DefaultSchedule() + s.State.Paused = true + return s + }, + maxSteps: 15, + }, + { + name: "limited actions exhausted (idles then closes)", + schedule: func() *schedulepb.Schedule { + s := schedulertest.DefaultSchedule() + s.State.LimitedActions = true + s.State.RemainingActions = 0 + return s + }, + maxSteps: 10, + }, + { + name: "limited actions, a few then idle", + schedule: func() *schedulepb.Schedule { + s := schedulertest.DefaultSchedule() + s.State.LimitedActions = true + s.State.RemainingActions = 3 + return s + }, + maxSteps: 30, + }, + { + name: "short interval, long catchup", + schedule: func() *schedulepb.Schedule { + s := schedulertest.DefaultSchedule() + s.Spec.Interval[0].Interval = durationpb.New(10 * time.Second) + return s + }, + maxSteps: 40, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + d := schedulertest.NewDriver(t, schedulertest.WithSchedule(tc.schedule())) + d.AfterStep = schedulertest.CheckInvariantsHook(t) + // The hook fails the test on any violation; we just drive it forward. + d.RunToQuiescence(tc.maxSteps) + }) + } +} + +// TestCheckInvariants_DetectsStuck is a direct unit test of the invariant +// predicate: a non-closed schedule with no pending task that is not held open is +// flagged as stuck. +func TestCheckInvariants_DetectsStuck(t *testing.T) { + stuck := schedulertest.Snapshot{ + Now: time.Now(), + HasPendingTask: false, + Closed: false, + IsHeldOpen: false, + } + violations := schedulertest.CheckInvariants(nil, stuck) + require.NotEmpty(t, violations, "stuck state must be flagged") + require.Equal(t, "no-stuck", violations[0].Name) +} + +// TestCheckInvariants_AllowsHeldOpenWithoutTask confirms a paused schedule with +// no pending task is NOT flagged (legitimate hold-open). +func TestCheckInvariants_AllowsHeldOpenWithoutTask(t *testing.T) { + heldOpen := schedulertest.Snapshot{ + Now: time.Now(), + HasPendingTask: false, + Closed: false, + Paused: true, + IsHeldOpen: true, + } + require.Empty(t, schedulertest.CheckInvariants(nil, heldOpen)) +} + +// TestCheckInvariants_DetectsHwmRegression confirms the high-water-mark +// monotonicity invariant fires when the generator HWM moves backward. +func TestCheckInvariants_DetectsHwmRegression(t *testing.T) { + t0 := time.Date(2020, 1, 1, 0, 5, 0, 0, time.UTC) + prev := schedulertest.Snapshot{HasPendingTask: true, GeneratorLPT: t0} + cur := schedulertest.Snapshot{HasPendingTask: true, GeneratorLPT: t0.Add(-time.Minute)} + violations := schedulertest.CheckInvariants(&prev, cur) + require.NotEmpty(t, violations) + require.Equal(t, "hwm-monotonic-generator", violations[0].Name) +} + +// TestCheckInvariants_DetectsBufferOverrun confirms the buffer-bound invariant +// fires when buffered starts exceed MaxBufferSize, and that being exactly at the +// cap is allowed. +func TestCheckInvariants_DetectsBufferOverrun(t *testing.T) { + over := schedulertest.Snapshot{HasPendingTask: true, MaxBufferSize: 1000, BufferedStartsCount: 1001} + violations := schedulertest.CheckInvariants(nil, over) + require.NotEmpty(t, violations, "buffer overrun must be flagged") + require.Equal(t, "buffer-bound", violations[0].Name) + + atCap := schedulertest.Snapshot{HasPendingTask: true, MaxBufferSize: 1000, BufferedStartsCount: 1000} + require.Empty(t, schedulertest.CheckInvariants(nil, atCap), "exactly at the cap is allowed") +} + +// TestCheckInvariants_DetectsBudgetOverrun confirms the action-budget invariant +// fires when a single ExecuteTask takes more actions than MaxActionsPerExecution +// (the double-spend bug class), and that being exactly at the cap is allowed. +func TestCheckInvariants_DetectsBudgetOverrun(t *testing.T) { + over := schedulertest.Snapshot{HasPendingTask: true, MaxActionsPerExecution: 5, MaxActionsObserved: 6} + violations := schedulertest.CheckInvariants(nil, over) + require.NotEmpty(t, violations, "budget overrun must be flagged") + require.Equal(t, "action-budget", violations[0].Name) + + atCap := schedulertest.Snapshot{HasPendingTask: true, MaxActionsPerExecution: 5, MaxActionsObserved: 5} + require.Empty(t, schedulertest.CheckInvariants(nil, atCap), "exactly at the cap is allowed") +} diff --git a/chasm/lib/scheduler/schedulertest/ledger_test.go b/chasm/lib/scheduler/schedulertest/ledger_test.go new file mode 100644 index 00000000000..b930f99f84d --- /dev/null +++ b/chasm/lib/scheduler/schedulertest/ledger_test.go @@ -0,0 +1,206 @@ +package schedulertest_test + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + enumspb "go.temporal.io/api/enums/v1" + schedulepb "go.temporal.io/api/schedule/v1" + "go.temporal.io/server/chasm/lib/scheduler" + "go.temporal.io/server/chasm/lib/scheduler/schedulertest" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// configWith returns DefaultConfig with Tweakables adjusted by fn. +func configWith(fn func(*scheduler.Tweakables)) *scheduler.Config { + cfg := schedulertest.DefaultConfig() + tweakables := scheduler.DefaultTweakables + fn(&tweakables) + cfg.Tweakables = func(string) scheduler.Tweakables { return tweakables } + return cfg +} + +// oneShotSchedule fires exactly once, at start+offset. +func oneShotSchedule(start time.Time, offset time.Duration) *schedulepb.Schedule { + s := schedulertest.DefaultSchedule() + s.Spec.Interval[0].Interval = durationpb.New(time.Hour) + s.Spec.Interval[0].Phase = durationpb.New(offset % time.Hour) + s.Spec.StartTime = timestamppb.New(start.Add(offset)) + s.Spec.EndTime = timestamppb.New(start.Add(offset + time.Minute)) + return s +} + +// collectViolations records every invariant violation instead of failing, so a +// test can assert that a known bug is detected. +func collectViolations(out *[]schedulertest.Violation) func(*schedulertest.Driver) { + var prev *schedulertest.Snapshot + return func(d *schedulertest.Driver) { + cur := d.Snapshot() + *out = append(*out, schedulertest.CheckInvariants(prev, cur)...) + prev = &cur + } +} + +func violationNames(vs []schedulertest.Violation) []string { + names := make([]string, 0, len(vs)) + for _, v := range vs { + names = append(names, v.Name) + } + return names +} + +// The final action of a schedule is buffered by the same generator tick that +// arms the idle task. If InvokerExecuteTask hasn't run by the time the idle task +// fires, the schedule must stay open. Pre-#13016 the idle deadline ignored the +// pending start and was already in the past. +func TestLedger_FinalStartPendingAtIdle(t *testing.T) { + const idle = time.Hour + start := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) + + cases := []struct { + name string + fixOn bool + offset time.Duration + wantBad []string + }{ + {"fix on, final gap > IdleTime", true, 3 * idle, nil}, + {"fix on, final gap < IdleTime", true, idle / 2, nil}, + {"fix off, final gap > IdleTime", false, 3 * idle, []string{"ledger-lost-start"}}, + {"fix off, final gap < IdleTime", false, idle / 2, nil}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var violations []schedulertest.Violation + d := schedulertest.NewDriver(t, + schedulertest.WithStartTime(start), + schedulertest.WithSchedule(oneShotSchedule(start, tc.offset)), + schedulertest.WithConfig(configWith(func(tw *scheduler.Tweakables) { + tw.IdleTime = idle + tw.EnablePendingStartIdleFix = tc.fixOn + })), + schedulertest.WithMaxSideEffectLag(idle/2), + schedulertest.WithAfterStep(collectViolations(&violations)), + ) + + // Fire timers while InvokerExecuteTask lags, then drain. + for d.StepPure() { + } + _, quiescent := d.RunToQuiescence(50) + require.True(t, quiescent) + + require.Equal(t, tc.wantBad, uniq(violationNames(violations))) + snap := d.Snapshot() + require.True(t, snap.Closed) + if tc.wantBad == nil { + require.Len(t, snap.StartRequested, 1, "the one action must start") + } + }) + } +} + +// A buffered start that vanishes with no RPC and no recorded discard is a +// silent drop; one with a recorded discard is not. +func TestCheckInvariants_Ledger(t *testing.T) { + pending := func(ids ...string) map[string]struct{} { + m := make(map[string]struct{}, len(ids)) + for _, id := range ids { + m[id] = struct{}{} + } + return m + } + + cases := []struct { + name string + prev, cur schedulertest.Snapshot + want []string + }{ + { + name: "started", + prev: schedulertest.Snapshot{HasPendingTask: true, PendingStarts: pending("a")}, + cur: schedulertest.Snapshot{HasPendingTask: true, StartRequested: pending("a")}, + }, + { + name: "overlap skipped", + prev: schedulertest.Snapshot{HasPendingTask: true, PendingStarts: pending("a")}, + cur: schedulertest.Snapshot{HasPendingTask: true, OverlapSkipped: 1}, + }, + { + name: "dropped", + prev: schedulertest.Snapshot{HasPendingTask: true, PendingStarts: pending("a", "b"), DroppedStarts: 3}, + cur: schedulertest.Snapshot{HasPendingTask: true, DroppedStarts: 4, StartRequested: pending("b")}, + }, + { + name: "silent drop", + prev: schedulertest.Snapshot{HasPendingTask: true, PendingStarts: pending("a", "b")}, + cur: schedulertest.Snapshot{HasPendingTask: true, OverlapSkipped: 1}, + want: []string{"ledger-silent-drop"}, + }, + { + name: "closed with pending start", + prev: schedulertest.Snapshot{HasPendingTask: true, PendingStarts: pending("a")}, + cur: schedulertest.Snapshot{Closed: true, PendingStarts: pending("a")}, + want: []string{"ledger-lost-start"}, + }, + { + name: "migration takes the buffer", + prev: schedulertest.Snapshot{HasPendingTask: true, PendingStarts: pending("a")}, + cur: schedulertest.Snapshot{Closed: true, Migrating: true, PendingStarts: pending("a")}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := schedulertest.CheckInvariants(&tc.prev, tc.cur) + require.Equal(t, tc.want, uniq(violationNames(got))) + }) + } +} + +func uniq(names []string) []string { + var out []string + seen := map[string]bool{} + for _, n := range names { + if !seen[n] { + seen[n] = true + out = append(out, n) + } + } + return out +} + +// A start deferred behind a running workflow is only re-enabled by a workflow +// completion. If that completion arrives while the schedule is paused, +// ProcessBuffer drops the first start (paused) and re-defers the rest with +// nothing running, so no completion will ever re-enable them. Unpause doesn't +// re-process the buffer either; the starts (including a manual trigger, which +// pause allows) sit until the idle task closes the schedule. +func TestLedger_DeferredStartsStrandedByPausedCompletion(t *testing.T) { + t.Skip("known bug: deferred starts stranded after a completion while paused") + + const idle = time.Hour + start := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) + s := schedulertest.DefaultSchedule() + s.Policies.OverlapPolicy = enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ALL + s.Spec.EndTime = timestamppb.New(start.Add(2*time.Minute + time.Second)) + + var violations []schedulertest.Violation + d := schedulertest.NewDriver(t, + schedulertest.WithStartTime(start), + schedulertest.WithSchedule(s), + schedulertest.WithConfig(configWith(func(tw *scheduler.Tweakables) { tw.IdleTime = idle })), + schedulertest.WithAfterStep(collectViolations(&violations)), + ) + + require.True(t, d.Step()) // 00:01 starts, keeps running + require.True(t, d.Step()) // 00:02 final fire, deferred behind it + require.NoError(t, d.Pause()) + require.NoError(t, d.TriggerImmediately()) // manual, deferred behind it + require.True(t, d.CompleteWorkflow()) + require.NoError(t, d.Unpause()) + _, quiescent := d.RunToQuiescence(50) + require.True(t, quiescent) + + require.Empty(t, violationNames(violations)) + require.Len(t, d.Snapshot().StartRequested, 2, "the manual trigger must start") +} diff --git a/chasm/lib/scheduler/schedulertest/library.go b/chasm/lib/scheduler/schedulertest/library.go new file mode 100644 index 00000000000..e8b827e321b --- /dev/null +++ b/chasm/lib/scheduler/schedulertest/library.go @@ -0,0 +1,174 @@ +// Package schedulertest provides a lifecycle-driving harness for the CHASM +// scheduler. Unlike the per-handler unit tests in package scheduler_test, this +// package builds a real component on top of [chasmtest.Engine] and steps it +// forward through every task it schedules for itself, so that invariants can be +// asserted across sequences of transitions (e.g. "the scheduler never gets +// stuck without a task to drive the next state"). +// +// It deliberately lives in a non-test package so that property tests +// (pgregory.net/rapid) and the V1<->V2 migration round-trip test can import the +// driver. The helpers below mirror chasm/lib/scheduler/helper_test.go, but are +// exported and accept the mock clients the side-effect task handlers need. +package schedulertest + +import ( + "time" + + commonpb "go.temporal.io/api/common/v1" + schedulepb "go.temporal.io/api/schedule/v1" + workflowpb "go.temporal.io/api/workflow/v1" + "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/server/chasm/lib/scheduler" + "go.temporal.io/server/common/backoff" + "go.temporal.io/server/common/clock" + "go.temporal.io/server/common/dynamicconfig" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/resource" + "go.temporal.io/server/common/searchattribute" + legacyscheduler "go.temporal.io/server/service/worker/scheduler" + "go.uber.org/mock/gomock" + "google.golang.org/protobuf/types/known/durationpb" +) + +const ( + Namespace = "ns" + NamespaceID = "ns-id" + ScheduleID = "sched-id" + + DefaultInterval = 1 * time.Minute + DefaultCatchupWindow = 5 * time.Minute +) + +// DefaultSchedule returns a schedule with a single 1-minute interval and a +// 5-minute catchup window, matching the package's other defaults. +func DefaultSchedule() *schedulepb.Schedule { + return &schedulepb.Schedule{ + Spec: &schedulepb.ScheduleSpec{ + Interval: []*schedulepb.IntervalSpec{ + { + Interval: durationpb.New(DefaultInterval), + Phase: durationpb.New(0), + }, + }, + }, + Action: &schedulepb.ScheduleAction{ + Action: &schedulepb.ScheduleAction_StartWorkflow{ + StartWorkflow: &workflowpb.NewWorkflowExecutionInfo{ + WorkflowId: "scheduled-wf", + WorkflowType: &commonpb.WorkflowType{Name: "scheduled-wf-type"}, + }, + }, + }, + Policies: &schedulepb.SchedulePolicies{ + CatchupWindow: durationpb.New(DefaultCatchupWindow), + }, + State: &schedulepb.ScheduleState{ + Paused: false, + LimitedActions: false, + RemainingActions: 0, + }, + } +} + +// DefaultConfig returns a scheduler config wired with the package defaults. +func DefaultConfig() *scheduler.Config { + return &scheduler.Config{ + Tweakables: func(_ string) scheduler.Tweakables { + return scheduler.DefaultTweakables + }, + ServiceCallTimeout: func() time.Duration { + return 5 * time.Second + }, + EncodeInternalTokenWithEnvelope: func(string) bool { + return true + }, + RetryPolicy: func() backoff.RetryPolicy { + return backoff.NewExponentialRetryPolicy(1 * time.Second) + }, + } +} + +// NewRealSpecProcessor builds a real SpecProcessor with no-op metrics, matching +// production spec processing so generated buffered starts and next-times are +// realistic. +func NewRealSpecProcessor(ctrl *gomock.Controller, logger log.Logger) scheduler.SpecProcessor { + mockMetrics := metrics.NewMockHandler(ctrl) + mockMetrics.EXPECT().Counter(gomock.Any()).Return(metrics.NoopCounterMetricFunc).AnyTimes() + mockMetrics.EXPECT().WithTags(gomock.Any()).Return(mockMetrics).AnyTimes() + mockMetrics.EXPECT().Timer(gomock.Any()).Return(metrics.NoopTimerMetricFunc).AnyTimes() + + return scheduler.NewSpecProcessor( + DefaultConfig(), + mockMetrics, + logger, + legacyscheduler.NewSpecBuilder( + dynamicconfig.SchedulerSpecWarnIterations.Get(dynamicconfig.NewNoopCollection()), + dynamicconfig.SchedulerSpecMaxIterations.Get(dynamicconfig.NewNoopCollection()), + ), + ) +} + +// NewTestLibrary builds a scheduler Library with all task handlers wired to the +// supplied spec processor and mock clients. The clients back the side-effect +// task handlers (execute/callbacks/migrate); pass mocks whose StartWorkflow / +// Terminate / Cancel expectations the test controls. +func NewTestLibrary( + config *scheduler.Config, + logger log.Logger, + metricsHandler metrics.Handler, + timeSource clock.TimeSource, + specProcessor scheduler.SpecProcessor, + frontendClient workflowservice.WorkflowServiceClient, + historyClient resource.HistoryClient, +) *scheduler.Library { + specBuilder := legacyscheduler.NewSpecBuilder( + dynamicconfig.SchedulerSpecWarnIterations.Get(dynamicconfig.NewNoopCollection()), + dynamicconfig.SchedulerSpecMaxIterations.Get(dynamicconfig.NewNoopCollection()), + ) + invokerOpts := scheduler.InvokerTaskHandlerOptions{ + Config: config, + MetricsHandler: metricsHandler, + BaseLogger: logger, + SpecProcessor: specProcessor, + HistoryClient: historyClient, + FrontendClient: frontendClient, + TimeSource: timeSource, + } + return scheduler.NewLibrary( + config, + nil, + scheduler.NewSchedulerIdleTaskHandler(scheduler.SchedulerIdleTaskHandlerOptions{ + Config: config, + MetricsHandler: metrics.NoopMetricsHandler, + BaseLogger: logger, + }), + scheduler.NewSchedulerCallbacksTaskHandler(scheduler.SchedulerCallbacksTaskHandlerOptions{ + Config: config, + HistoryClient: historyClient, + FrontendClient: frontendClient, + }), + scheduler.NewGeneratorTaskHandler(scheduler.GeneratorTaskHandlerOptions{ + Config: config, + MetricsHandler: metrics.NoopMetricsHandler, + BaseLogger: logger, + SpecProcessor: specProcessor, + SpecBuilder: specBuilder, + }), + scheduler.NewInvokerExecuteTaskHandler(invokerOpts), + scheduler.NewInvokerProcessBufferTaskHandler(invokerOpts), + scheduler.NewBackfillerTaskHandler(scheduler.BackfillerTaskHandlerOptions{ + Config: config, + MetricsHandler: metrics.NoopMetricsHandler, + BaseLogger: logger, + SpecProcessor: specProcessor, + }), + scheduler.NewSchedulerMigrateToWorkflowTaskHandler(scheduler.SchedulerMigrateToWorkflowTaskHandlerOptions{ + Config: config, + MetricsHandler: metrics.NoopMetricsHandler, + BaseLogger: logger, + HistoryClient: historyClient, + SaMapperProvider: searchattribute.NewTestMapperProvider(nil), + }), + ) +} diff --git a/chasm/lib/scheduler/schedulertest/property_test.go b/chasm/lib/scheduler/schedulertest/property_test.go new file mode 100644 index 00000000000..02f1bf53133 --- /dev/null +++ b/chasm/lib/scheduler/schedulertest/property_test.go @@ -0,0 +1,155 @@ +package schedulertest_test + +import ( + "testing" + "time" + + enumspb "go.temporal.io/api/enums/v1" + schedulepb "go.temporal.io/api/schedule/v1" + "go.temporal.io/server/chasm/lib/scheduler" + "go.temporal.io/server/chasm/lib/scheduler/schedulertest" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" + "pgregory.net/rapid" +) + +// TestSchedulerProperties drives a randomly-generated schedule through a random +// sequence of operations (advance time, pause, unpause, trigger) and asserts the +// scheduler invariants hold after every settled transition. rapid shrinks any +// failing operation sequence to a minimal reproducer — directly useful for the +// "stuck open" bug class. +// +// The outer *testing.T owns the engine scaffolding (it requires a real +// testing.TB); invariant violations are reported through the *rapid.T so rapid +// can shrink them. +func TestSchedulerProperties(t *testing.T) { + rapid.Check(t, func(rt *rapid.T) { + start := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) + // A small IdleTime lets fire-time gaps cross the idle window, which the + // 7d default never does with these intervals. + idle := rapid.SampledFrom([]time.Duration{10 * time.Minute, time.Hour, 2 * time.Hour}).Draw(rt, "idleTime") + sched := drawSchedule(rt, start, idle) + + d := schedulertest.NewDriver(t, + schedulertest.WithStartTime(start), + schedulertest.WithSchedule(sched), + schedulertest.WithConfig(configWith(func(tw *scheduler.Tweakables) { + tw.IdleTime = idle + })), + // InvokerExecuteTask may lag the timers, and workflows may run, but + // neither for a whole idle window. Past that, V1 and V2 both close + // with the start still buffered. + schedulertest.WithMaxSideEffectLag(idle/2), + schedulertest.WithMaxRunDuration(idle/2), + ) + hook := checkInvariantsIgnoringKnownBugs(rt) + d.AfterStep = hook + + // Check the initial settled state too. + hook(d) + + numOps := rapid.IntRange(1, 60).Draw(rt, "numOps") + for range numOps { + // Bias toward stepping so the clock advances and the schedule actually + // progresses through generator/invoker/idle transitions. "pure" fires + // timers without dispatching side-effect tasks, so InvokerExecuteTask + // can lag behind the generator and idle tasks. + op := rapid.SampledFrom([]string{ + "step", "step", "pure", "pure", "sideEffect", "complete", + "pause", "unpause", "trigger", + }).Draw(rt, "op") + + switch op { + case "step": + d.Step() + case "pure": + d.StepPure() + case "sideEffect": + d.StepSideEffect() + case "complete": + d.CompleteWorkflow() + case "pause": + _ = d.Pause() + case "unpause": + _ = d.Unpause() + case "trigger": + _ = d.TriggerImmediately() + default: + rt.Fatalf("unhandled op %q", op) + } + } + }) +} + +// drawSchedule generates a random but valid schedule definition. Some +// schedules end, with their last fire time placed around the idle window. +func drawSchedule(rt *rapid.T, start time.Time, idle time.Duration) *schedulepb.Schedule { + s := schedulertest.DefaultSchedule() + + intervalSec := rapid.SampledFrom([]int64{10, 30, 60, 300, 3600, 3 * 3600}).Draw(rt, "intervalSec") + interval := time.Duration(intervalSec) * time.Second + s.Spec.Interval[0].Interval = durationpb.New(interval) + + if jitterSec := rapid.SampledFrom([]int64{0, 1, 5, 30}).Draw(rt, "jitterSec"); jitterSec > 0 { + s.Spec.Jitter = durationpb.New(time.Duration(jitterSec) * time.Second) + } + + if catchupSec := rapid.SampledFrom([]int64{60, 300, 3600}).Draw(rt, "catchupSec"); catchupSec > 0 { + s.Policies.CatchupWindow = durationpb.New(time.Duration(catchupSec) * time.Second) + } + + s.Policies.OverlapPolicy = rapid.SampledFrom([]enumspb.ScheduleOverlapPolicy{ + enumspb.SCHEDULE_OVERLAP_POLICY_SKIP, + enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ONE, + enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ALL, + enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL, + }).Draw(rt, "overlapPolicy") + + switch rapid.SampledFrom([]string{"forever", "endTime", "oneShot"}).Draw(rt, "shape") { + case "endTime": + // Last fire lands a few intervals in, so the final gap is the interval. + n := rapid.IntRange(1, 4).Draw(rt, "firesBeforeEnd") + s.Spec.EndTime = timestamppb.New(start.Add(time.Duration(n)*interval + time.Second)) + case "oneShot": + // A single fire, before, at, or after the idle window since creation. + offset := rapid.SampledFrom([]time.Duration{ + idle / 2, idle - time.Minute, idle, idle + time.Minute, 3 * idle, + }).Draw(rt, "oneShotOffset") + s.Spec.Interval[0].Interval = durationpb.New(time.Hour) + s.Spec.Interval[0].Phase = durationpb.New(offset % time.Hour) + s.Spec.StartTime = timestamppb.New(start.Add(offset)) + s.Spec.EndTime = timestamppb.New(start.Add(offset + time.Minute)) + default: + } + + if rapid.Bool().Draw(rt, "paused") { + s.State.Paused = true + } + if rapid.Bool().Draw(rt, "limited") { + s.State.LimitedActions = true + s.State.RemainingActions = int64(rapid.IntRange(0, 5).Draw(rt, "remainingActions")) + } + + return s +} + +// checkInvariantsIgnoringKnownBugs is CheckInvariantsHook minus one known bug +// (TestLedger_DeferredStartsStrandedByPausedCompletion): a lost start where +// every pending start is deferred and nothing is running to re-enable them. +// Delete this once that bug is fixed. +func checkInvariantsIgnoringKnownBugs(rt *rapid.T) func(*schedulertest.Driver) { + var prev *schedulertest.Snapshot + return func(d *schedulertest.Driver) { + rt.Helper() + cur := d.Snapshot() + for _, v := range schedulertest.CheckInvariants(prev, cur) { + if v.Name == "ledger-lost-start" && cur.RunningWorkflows == 0 && + cur.DeferredStarts == len(cur.PendingStarts) { + continue + } + rt.Fatalf("scheduler invariant violation at step (now=%s): %s", + cur.Now.Format("2006-01-02T15:04:05"), v) + } + prev = &cur + } +} diff --git a/go.mod b/go.mod index a013ef821bb..fca0a4545b2 100644 --- a/go.mod +++ b/go.mod @@ -93,6 +93,7 @@ require ( gopkg.in/validator.v2 v2.0.1 gopkg.in/yaml.v3 v3.0.1 modernc.org/sqlite v1.51.0 + pgregory.net/rapid v1.3.0 ) require ( diff --git a/go.sum b/go.sum index 03801765900..af9ae6c5003 100644 --- a/go.sum +++ b/go.sum @@ -699,6 +699,8 @@ modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +pgregory.net/rapid v1.3.0 h1:vBvO0VSqti75J1jjYqpgPNBLKMd1+gxa9fYo7vk/Exc= +pgregory.net/rapid v1.3.0/go.mod h1:dPlE4OBBxgXPqkP79flB6sJL1dx5azpI7HQ9MY9Z7uk= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= From 780fc976351aaa80ddb6022de7235d91a8e2fc04 Mon Sep 17 00:00:00 2001 From: chaptersix <13949480+chaptersix@users.noreply.github.com> Date: Wed, 11 Nov 2026 15:13:23 -0600 Subject: [PATCH 02/22] Strengthen scheduler lifecycle checks and fix exposed progress bugs Separate external acceptance, RPC responses, callbacks, and local result recording. Check accepted-start accounting, logical task validity, idle routes, retry identity, and occurrence accounting; retain deterministic regressions and mutation checks. Preserve pending ALLOW_ALL starts when callbacks precede recording, release deferred work after selected starts are discarded, and admit backfills when positive shared capacity divides below one. Build on temporalio/temporal#13042. Reuse and reference chaptersix/temporal#35, #41, #42, #43, #44, #45, #51, #52, #53, #55, #56, #57, #59, #62, and #70. Keep unresolved migration counterexamples opt-in and document the remaining reservation and persistence gaps. --- .../backfill_capacity_progress_test.go | 69 +++++ .../backfiller_capacity_internal_test.go | 4 +- chasm/lib/scheduler/backfiller_tasks.go | 3 +- chasm/lib/scheduler/completion.go | 10 + chasm/lib/scheduler/invoker.go | 12 + .../lib/scheduler/migration_backfill_test.go | 185 ++++++++++++ .../migration_chain_counterexample_test.go | 55 ++++ chasm/lib/scheduler/migration_handoff_test.go | 169 +++++++++++ .../migration_rollback_phase_test.go | 89 ++++++ chasm/lib/scheduler/result_reordering_test.go | 33 +++ .../schedulertest/check_mutations.py | 46 +++ chasm/lib/scheduler/schedulertest/driver.go | 164 ++++++++--- chasm/lib/scheduler/schedulertest/external.go | 231 +++++++++++++++ .../scheduler/schedulertest/external_test.go | 137 +++++++++ .../scheduler/schedulertest/inflight_test.go | 80 +++++ .../lib/scheduler/schedulertest/invariants.go | 36 ++- .../scheduler/schedulertest/ledger_test.go | 4 +- .../schedulertest/planner_properties_test.go | 37 +++ .../scheduler/schedulertest/property_test.go | 31 +- .../scheduler/schedulertest/task_validity.go | 38 +++ .../schedulertest/task_validity_test.go | 108 +++++++ ...edulerProperties-20261111145706-76678.fail | 45 +++ .../spec_processor_properties_test.go | 199 +++++++++++++ .../scheduler/task_validity_engine_test.go | 276 ++++++++++++++++++ .../chasm-schedulertest-coverage-plan.md | 116 ++++++++ .../frontend/migration_delete_race_test.go | 103 +++++++ .../scheduler/migration_scenarios_test.go | 271 +++++++++++++++++ 27 files changed, 2482 insertions(+), 69 deletions(-) create mode 100644 chasm/lib/scheduler/backfill_capacity_progress_test.go create mode 100644 chasm/lib/scheduler/migration_backfill_test.go create mode 100644 chasm/lib/scheduler/migration_chain_counterexample_test.go create mode 100644 chasm/lib/scheduler/migration_handoff_test.go create mode 100644 chasm/lib/scheduler/migration_rollback_phase_test.go create mode 100644 chasm/lib/scheduler/result_reordering_test.go create mode 100644 chasm/lib/scheduler/schedulertest/check_mutations.py create mode 100644 chasm/lib/scheduler/schedulertest/external.go create mode 100644 chasm/lib/scheduler/schedulertest/external_test.go create mode 100644 chasm/lib/scheduler/schedulertest/inflight_test.go create mode 100644 chasm/lib/scheduler/schedulertest/planner_properties_test.go create mode 100644 chasm/lib/scheduler/schedulertest/task_validity.go create mode 100644 chasm/lib/scheduler/schedulertest/task_validity_test.go create mode 100644 chasm/lib/scheduler/schedulertest/testdata/rapid/TestSchedulerProperties/TestSchedulerProperties-20261111145706-76678.fail create mode 100644 chasm/lib/scheduler/spec_processor_properties_test.go create mode 100644 chasm/lib/scheduler/task_validity_engine_test.go create mode 100644 docs/architecture/chasm-schedulertest-coverage-plan.md create mode 100644 service/frontend/migration_delete_race_test.go create mode 100644 service/worker/scheduler/migration_scenarios_test.go diff --git a/chasm/lib/scheduler/backfill_capacity_progress_test.go b/chasm/lib/scheduler/backfill_capacity_progress_test.go new file mode 100644 index 00000000000..8a8c5b2e8e4 --- /dev/null +++ b/chasm/lib/scheduler/backfill_capacity_progress_test.go @@ -0,0 +1,69 @@ +package scheduler_test + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + enumspb "go.temporal.io/api/enums/v1" + schedulepb "go.temporal.io/api/schedule/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/scheduler" + "go.temporal.io/server/common/clock" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestBackfillCapacityNativeControl(t *testing.T) { + testBackfillCapacityProgress(t, 450) +} + +func TestBackfillCapacityProgress(t *testing.T) { + for _, n := range []int{451, 1000} { + t.Run(fmt.Sprint(n), func(t *testing.T) { testBackfillCapacityProgress(t, n) }) + } +} + +func testBackfillCapacityProgress(t *testing.T, count int) { + t.Helper() + now := time.Date(2026, 9, 4, 12, 0, 0, 0, time.UTC) + ts := clock.NewEventTimeSource().Update(now) + spec := defaultSchedule() + spec.State.Paused = true + e := newSchedulerTestEngine(t, spec, withEngineTimeSource(ts)) + require.NoError(t, e.updateScheduler(func(s *scheduler.Scheduler, ctx chasm.MutableContext) error { + for range count { + s.NewRangeBackfiller(ctx, &schedulepb.BackfillRequest{ + StartTime: timestamppb.New(now), EndTime: timestamppb.New(now), + OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL, + }) + } + return nil + })) + seen := make(map[string]string) + remaining := count + for round := 0; round < 50 && remaining > 0; round++ { + buffered := 0 + require.NoError(t, e.updateScheduler(func(s *scheduler.Scheduler, ctx chasm.MutableContext) error { + i := s.Invoker.Get(ctx) + buffered = len(i.BufferedStarts) + require.LessOrEqual(t, buffered, 460, "shared half-buffer minus generator reserve plus retained-history allowance") + for _, start := range i.BufferedStarts { + require.NotContains(t, seen, start.RequestId) + seen[start.RequestId] = start.WorkflowId + } + i.BufferedStarts = nil + remaining = len(s.Backfillers) + return nil + })) + if remaining == 0 { + break + } + require.Positive(t, buffered, "seed=capacity-%d: empty buffer and %d ranges must make progress", count, remaining) + ts.Update(ts.Now().Add(time.Hour)) + _, err := e.engine.FirePureTasks(e.rootRef, ts.Now()) + require.NoError(t, err) + } + require.Zero(t, remaining) + require.Len(t, seen, count) +} diff --git a/chasm/lib/scheduler/backfiller_capacity_internal_test.go b/chasm/lib/scheduler/backfiller_capacity_internal_test.go index c697aa0019b..29e852ad6b8 100644 --- a/chasm/lib/scheduler/backfiller_capacity_internal_test.go +++ b/chasm/lib/scheduler/backfiller_capacity_internal_test.go @@ -28,7 +28,7 @@ import ( // // pending = max(0, bufferedCount - retainedActionCount) // available = max(0, maxBufferSize/2 - pending - generatorReserve) -// result = available / max(1, backfillerCount) +// result = min(available, max(1, available / max(1, backfillerCount))) func TestBackfillerBufferCapacity(t *testing.T) { cases := []struct { name string @@ -77,7 +77,7 @@ func TestBackfillerBufferCapacity(t *testing.T) { {"single backfiller gets the whole remainder", 0, 0, 1000, 50, 1, 450}, {"ten backfillers each get an even share (regression)", 0, 0, 1000, 50, 10, 45}, {"backfillers exactly dividing remainder get one each", 0, 0, 1000, 50, 450, 1}, - {"more backfillers than remainder truncate to zero", 0, 0, 1000, 50, 451, 0}, + {"more backfillers than remainder still progress", 0, 0, 1000, 50, 451, 1}, {"zero backfillers are clamped to one", 0, 0, 1000, 50, 0, 450}, {"negative backfillers are clamped to one", 0, 0, 1000, 50, -3, 450}, } diff --git a/chasm/lib/scheduler/backfiller_tasks.go b/chasm/lib/scheduler/backfiller_tasks.go index 082215cbfcc..37ee41d3134 100644 --- a/chasm/lib/scheduler/backfiller_tasks.go +++ b/chasm/lib/scheduler/backfiller_tasks.go @@ -293,5 +293,6 @@ func backfillerBufferCapacity(bufferedCount, retainedActionCount, maxBufferSize, backfillerCount = max(1, backfillerCount) pending := max(0, bufferedCount-retainedActionCount) available := max(0, (maxBufferSize/2)-pending-generatorReserve) - return available / backfillerCount + // Pure tasks serialize on the tree and recompute the shared budget after each enqueue. + return min(available, max(1, available/backfillerCount)) } diff --git a/chasm/lib/scheduler/completion.go b/chasm/lib/scheduler/completion.go index 30208d05123..2ba8cc6ec03 100644 --- a/chasm/lib/scheduler/completion.go +++ b/chasm/lib/scheduler/completion.go @@ -20,6 +20,7 @@ import ( const ( callbackIgnoredUnrecognizedRequest metrics.ReasonString = "unrecognized_request_id" callbackIgnoredAlreadyCompleted metrics.ReasonString = "already_completed" + callbackIgnoredStartNotRecorded metrics.ReasonString = "start_not_recorded" ) var _ chasm.NexusCompletionHandler = &Scheduler{} @@ -127,6 +128,15 @@ func (s *Scheduler) HandleNexusCompletion( ) return nil } + if !internal.TracksCompletionResult(start.GetOverlapPolicy()) && start.GetRunId() == "" { + // A callback can beat the start RPC response. Preserve the pending entry + // until that response records its identity and action count. + metricsHandler.Counter(metrics.ScheduleCallbackIgnored.Name()).Record( + 1, + metrics.ReasonTag(callbackIgnoredStartNotRecorded), + ) + return nil + } if start.GetCompleted() != nil { // Completion callbacks may be validly redelivered, for example after a workflow reset. // Preserve state but keep the duplicate observable through the log and metric. diff --git a/chasm/lib/scheduler/invoker.go b/chasm/lib/scheduler/invoker.go index 3580a43d2ae..0cc6a35a773 100644 --- a/chasm/lib/scheduler/invoker.go +++ b/chasm/lib/scheduler/invoker.go @@ -123,6 +123,18 @@ func (i *Invoker) recordProcessBufferResult(ctx chasm.MutableContext, result *pr i.TerminateWorkflows = append(i.GetTerminateWorkflows(), result.terminateWorkflows...) i.LastProcessedTime = timestamppb.New(ctx.Now(i)) + // A discarded selection cannot produce the completion that would release + // deferred work. Reconsider it while each pass removes at least one start. + if len(discards) > 0 && !slices.ContainsFunc(starts, func(start *schedulespb.BufferedStart) bool { + return start.GetCompleted() == nil && (start.GetRunId() != "" || start.GetAttempt() > 0) + }) { + for _, start := range starts { + if start.GetAttempt() == -1 { + start.Attempt = 0 + } + } + } + // Re-arm tasks if this call changed state, or if the LastProcessedTime advance // just unblocked backed-off starts. i.addTasks(ctx) diff --git a/chasm/lib/scheduler/migration_backfill_test.go b/chasm/lib/scheduler/migration_backfill_test.go new file mode 100644 index 00000000000..e00c36741b1 --- /dev/null +++ b/chasm/lib/scheduler/migration_backfill_test.go @@ -0,0 +1,185 @@ +package scheduler_test + +import ( + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + schedulepb "go.temporal.io/api/schedule/v1" + workflowpb "go.temporal.io/api/workflow/v1" + "go.temporal.io/server/api/historyservice/v1" + "go.temporal.io/server/api/historyservicemock/v1" + schedulespb "go.temporal.io/server/api/schedule/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/scheduler" + schedulerpb "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" + "go.temporal.io/server/chasm/lib/scheduler/migration" + "go.temporal.io/server/common/clock" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/testing/protorequire" + "go.temporal.io/server/common/testing/testlogger" + "go.uber.org/mock/gomock" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestMigrationBackfill_NativeResumesAfterWatermark(t *testing.T) { + checkMigrationBackfillWatermark(t, false, time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC)) +} + +func TestMigrationBackfill_MigratedResumesAfterWatermark(t *testing.T) { + if os.Getenv("TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES") == "" { + t.Skip("set TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES=1 to run known failing migration repros") + } + for _, end := range []time.Time{time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC), time.Unix(60, 0).UTC()} { + t.Run(end.String(), func(t *testing.T) { checkMigrationBackfillWatermark(t, true, end) }) + } +} + +func checkMigrationBackfillWatermark(t *testing.T, migrate bool, when time.Time) { + t.Helper() + env := newTestEnv(t) + env.TimeSource.Update(when) + watermark := timestamppb.New(when.Add(-time.Minute)) + backfill := &schedulepb.BackfillRequest{ + StartTime: watermark, + EndTime: timestamppb.New(when), + OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL, + } + state := &schedulerpb.BackfillerState{ + Request: &schedulerpb.BackfillerState_BackfillRequest{BackfillRequest: backfill}, + LastProcessedTime: watermark, + Attempt: 1, + } + if migrate { + imported := migration.LegacyToCreateFromMigrationStateRequest( + env.Scheduler.Schedule, + &schedulepb.ScheduleInfo{ActionCount: 1}, + &schedulespb.InternalState{ + Namespace: namespace, + NamespaceId: namespaceID, + ScheduleId: scheduleID, + OngoingBackfills: []*schedulepb.BackfillRequest{backfill}, + }, + nil, + nil, + when, + ) + require.Len(t, imported.State.Backfillers, 1) + for _, converted := range imported.State.Backfillers { + state = converted + } + } + handler := scheduler.NewBackfillerTaskHandler(scheduler.BackfillerTaskHandlerOptions{ + Config: defaultConfig(), + MetricsHandler: metrics.NoopMetricsHandler, + BaseLogger: env.Logger, + SpecProcessor: env.SpecProcessor, + }) + result, err := handler.ProcessBackfill(env.Scheduler, &scheduler.Backfiller{BackfillerState: state}, 10) + require.NoError(t, err) + require.True(t, result.Complete) + require.Len(t, result.BufferedStarts, 1, "V1's incremental cursor is exclusive") + protorequire.ProtoEqual(t, timestamppb.New(when), result.BufferedStarts[0].ActualTime) +} + +func TestMigrationBackfill_WaitsForCallbackReconciliation(t *testing.T) { + requireMigrationCounterexamples(t) + boundary := time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC) + logger := testlogger.NewTestLogger(t, testlogger.FailOnExpectedErrorOnly) + timeSource := clock.NewEventTimeSource().Update(boundary.Add(time.Hour)) + engine, engineCtx := newTestEngineContext(t, logger, withEngineTimeSource(timeSource)) + schedule := defaultSchedule() + schedule.State.Paused = true + schedule.Policies.OverlapPolicy = enumspb.SCHEDULE_OVERLAP_POLICY_SKIP + request := migration.LegacyToCreateFromMigrationStateRequest( + schedule, + &schedulepb.ScheduleInfo{RunningWorkflows: []*commonpb.WorkflowExecution{{ + WorkflowId: "stale-workflow", + RunId: "stale-run", + }}}, + &schedulespb.InternalState{ + Namespace: namespace, + NamespaceId: namespaceID, + ScheduleId: scheduleID, + LastProcessedTime: timestamppb.New(timeSource.Now()), + BufferedStarts: []*schedulespb.BufferedStart{{ + NominalTime: timestamppb.New(boundary), + ActualTime: timestamppb.New(boundary), + Manual: true, + RequestId: "pending-request", + WorkflowId: "pending-workflow", + }}, + OngoingBackfills: []*schedulepb.BackfillRequest{{ + StartTime: timestamppb.New(boundary), + EndTime: timestamppb.New(boundary.Add(time.Minute)), + }}, + }, + nil, + nil, + timeSource.Now(), + ) + + _, err := scheduler.NewTestHandler(logger).TestCreateFromMigrationState(engineCtx, request) + require.NoError(t, err) + ref := chasm.NewComponentRef[*scheduler.Scheduler](chasm.ExecutionKey{ + NamespaceID: namespaceID, + BusinessID: scheduleID, + }) + _, err = chasm.ReadComponent( + engineCtx, + ref, + func(s *scheduler.Scheduler, ctx chasm.Context, _ struct{}) (struct{}, error) { + require.Len(t, s.Backfillers, 1) + require.Zero(t, s.Info.OverlapSkipped) + starts := s.Invoker.Get(ctx).BufferedStarts + require.Len(t, starts, 2) + require.Equal(t, "pending-request", starts[0].RequestId) + require.False(t, starts[1].HasCallback) + return struct{}{}, nil + }, + struct{}{}, + ) + require.NoError(t, err) + + historyClient := historyservicemock.NewMockHistoryServiceClient(gomock.NewController(t)) + historyClient.EXPECT().DescribeWorkflowExecution(gomock.Any(), gomock.Any()).Return( + &historyservice.DescribeWorkflowExecutionResponse{ + WorkflowExecutionInfo: &workflowpb.WorkflowExecutionInfo{ + Status: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, + CloseTime: timestamppb.New(boundary.Add(time.Minute)), + }, + }, + nil, + ) + callbackHandler := scheduler.NewSchedulerCallbacksTaskHandler(scheduler.SchedulerCallbacksTaskHandlerOptions{ + Config: defaultConfig(), + HistoryClient: historyClient, + }) + require.NoError(t, callbackHandler.Execute( + engineCtx, + ref, + chasm.TaskAttributes{}, + &schedulerpb.SchedulerCallbacksTask{}, + )) + _, err = engine.FirePureTasks(ref, timeSource.Now()) + require.NoError(t, err) + _, err = chasm.ReadComponent( + engineCtx, + ref, + func(s *scheduler.Scheduler, ctx chasm.Context, _ struct{}) (struct{}, error) { + require.Empty(t, s.Backfillers) + require.LessOrEqual(t, s.Info.OverlapSkipped, int64(1)) + for _, start := range s.Invoker.Get(ctx).BufferedStarts { + if start.GetRunId() == "stale-run" { + require.NotNil(t, start.Completed) + } + } + return struct{}{}, nil + }, + struct{}{}, + ) + require.NoError(t, err) +} diff --git a/chasm/lib/scheduler/migration_chain_counterexample_test.go b/chasm/lib/scheduler/migration_chain_counterexample_test.go new file mode 100644 index 00000000000..ed480e8a976 --- /dev/null +++ b/chasm/lib/scheduler/migration_chain_counterexample_test.go @@ -0,0 +1,55 @@ +package scheduler_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "go.temporal.io/api/serviceerror" + "go.temporal.io/server/api/historyservice/v1" + persistencespb "go.temporal.io/server/api/persistence/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/scheduler" + "go.uber.org/mock/gomock" + "google.golang.org/grpc" +) + +func TestMigrationCounterexampleRollbackChainOwnership(t *testing.T) { + requireMigrationCounterexamples(t) + for _, scenario := range []string{"owned_descendant", "foreign_chain", "first_run_removed"} { + t.Run(scenario, func(t *testing.T) { + e, handler, history := newRollbackScenario(t) + history.EXPECT().StartWorkflowExecution(gomock.Any(), gomock.Any()).Return(nil, + serviceerror.NewWorkflowExecutionAlreadyStarted("continued-as-new destination", "descendant-request", "descendant-run")) + history.EXPECT().GetMutableState(gomock.Any(), gomock.Any()).Return( + &historyservice.GetMutableStateResponse{FirstExecutionRunId: "first-run"}, nil).AnyTimes() + verified := false + history.EXPECT().DescribeMutableState(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, req *historyservice.DescribeMutableStateRequest, _ ...grpc.CallOption) (*historyservice.DescribeMutableStateResponse, error) { + require.Equal(t, "first-run", req.GetExecution().GetRunId()) + verified = true + if scenario == "first_run_removed" { + return nil, serviceerror.NewNotFound("original run expired") + } + owner := "rollback" + if scenario == "foreign_chain" { + owner = "foreign" + } + return &historyservice.DescribeMutableStateResponse{DatabaseMutableState: &persistencespb.WorkflowMutableState{ + ExecutionState: &persistencespb.WorkflowExecutionState{RunId: "first-run", CreateRequestId: owner}, + }}, nil + }).AnyTimes() + _, err := executeRollbackTask(t, e, handler, e.engine) + if scenario == "owned_descendant" { + require.NoError(t, err) + } else { + require.Error(t, err) + } + require.True(t, verified, "a descendant's start request does not establish ownership of the original migration") + require.NoError(t, e.readScheduler(func(s *scheduler.Scheduler, _ chasm.Context) error { + require.Equal(t, scenario == "owned_descendant", s.Closed) + return nil + })) + }) + } +} diff --git a/chasm/lib/scheduler/migration_handoff_test.go b/chasm/lib/scheduler/migration_handoff_test.go new file mode 100644 index 00000000000..735409ee902 --- /dev/null +++ b/chasm/lib/scheduler/migration_handoff_test.go @@ -0,0 +1,169 @@ +package scheduler_test + +import ( + "context" + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + schedulepb "go.temporal.io/api/schedule/v1" + "go.temporal.io/api/serviceerror" + "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/server/api/historyservice/v1" + "go.temporal.io/server/api/historyservicemock/v1" + schedulespb "go.temporal.io/server/api/schedule/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/scheduler" + schedulerpb "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" + "go.temporal.io/server/common/clock" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/payload" + "go.temporal.io/server/common/sdk" + "go.temporal.io/server/common/searchattribute" + "go.temporal.io/server/common/testing/protorequire" + "go.uber.org/mock/gomock" + "google.golang.org/grpc" +) + +type migrationCloseFaultEngine struct { + chasm.Engine + afterCommit bool + armed bool +} + +func requireMigrationCounterexamples(t *testing.T) { + t.Helper() + if os.Getenv("TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES") == "" { + t.Skip("set TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES=1 to run known failing migration repros") + } +} + +func (e *migrationCloseFaultEngine) UpdateComponent( + ctx context.Context, ref chasm.ComponentRef, update func(chasm.MutableContext, chasm.Component) error, opts ...chasm.TransitionOption, +) ([]byte, error) { + if !e.armed { + return e.Engine.UpdateComponent(ctx, ref, update, opts...) + } + e.armed = false + if e.afterCommit { + if _, err := e.Engine.UpdateComponent(ctx, ref, update, opts...); err != nil { + return nil, err + } + } + return nil, serviceerror.NewUnavailable("injected source-close response loss") +} + +func newRollbackScenario(t *testing.T) (*schedulerTestEngine, *scheduler.SchedulerMigrateToWorkflowTaskHandler, *historyservicemock.MockHistoryServiceClient) { + t.Helper() + ts := clock.NewEventTimeSource().Update(time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC)) + sched := defaultSchedule() + sched.State.Paused = true + sched.State.Notes = "user maintenance" + e := newSchedulerTestEngine(t, sched, withEngineTimeSource(ts)) + require.NoError(t, e.updateScheduler(func(s *scheduler.Scheduler, ctx chasm.MutableContext) error { + _, err := s.Patch(ctx, &schedulerpb.PatchScheduleRequest{FrontendRequest: &workflowservice.PatchScheduleRequest{ + RequestId: "trigger-request", Patch: &schedulepb.SchedulePatch{ + TriggerImmediately: &schedulepb.TriggerImmediatelyRequest{OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL}, + }, + }}) + if err != nil { + return err + } + s.Visibility.Get(ctx).MergeCustomMemo(ctx, map[string]*commonpb.Payload{"custom": payload.EncodeString("memo")}) + if err := s.Visibility.Get(ctx).MergeCustomSearchAttributes(ctx, map[string]*commonpb.Payload{"CustomKeywordField": payload.EncodeString("attribute")}); err != nil { + return err + } + _, err = s.MigrateToWorkflow(ctx, &schedulerpb.MigrateToWorkflowRequest{RequestId: "rollback"}) + return err + })) + historyClient := historyservicemock.NewMockHistoryServiceClient(gomock.NewController(t)) + handler := scheduler.NewSchedulerMigrateToWorkflowTaskHandler(scheduler.SchedulerMigrateToWorkflowTaskHandlerOptions{ + Config: defaultConfig(), MetricsHandler: metrics.NoopMetricsHandler, BaseLogger: e.logger, + HistoryClient: historyClient, SaMapperProvider: searchattribute.NewTestMapperProvider(nil), + }) + return e, handler, historyClient +} + +func executeRollbackTask(t *testing.T, e *schedulerTestEngine, handler *scheduler.SchedulerMigrateToWorkflowTaskHandler, engine chasm.Engine) (bool, error) { + t.Helper() + var valid bool + err := e.readScheduler(func(s *scheduler.Scheduler, ctx chasm.Context) error { + var err error + valid, err = handler.Validate(ctx, s, chasm.TaskInvocation{}, &schedulerpb.SchedulerMigrateToWorkflowTask{}) + return err + }) + if err != nil || !valid { + return !valid, err + } + return false, handler.Execute(chasm.NewEngineContext(context.Background(), engine), e.rootRef, chasm.TaskAttributes{}, &schedulerpb.SchedulerMigrateToWorkflowTask{}) +} + +func TestMigrationScenario_RollbackRetryBoundaries(t *testing.T) { + for _, boundary := range []string{"success", "before_destination", "after_destination", "before_source_close", "after_source_close"} { + t.Run(boundary, func(t *testing.T) { + e, handler, historyClient := newRollbackScenario(t) + faultEngine := &migrationCloseFaultEngine{Engine: e.engine, + armed: boundary == "before_source_close" || boundary == "after_source_close", afterCommit: boundary == "after_source_close"} + var destination *schedulespb.StartScheduleArgs + attempts, creations := 0, 0 + historyClient.EXPECT().StartWorkflowExecution(gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn( + func(_ context.Context, req *historyservice.StartWorkflowExecutionRequest, _ ...grpc.CallOption) (*historyservice.StartWorkflowExecutionResponse, error) { + attempts++ + if attempts == 1 && boundary == "before_destination" { + return nil, serviceerror.NewUnavailable("injected before destination commit") + } + if destination != nil { + return nil, serviceerror.NewWorkflowExecutionAlreadyStarted("retry", "first-request", "destination-run") + } + destination = &schedulespb.StartScheduleArgs{} + require.NoError(t, sdk.PreferProtoDataConverter.FromPayloads(req.StartRequest.Input, destination)) + protorequire.ProtoEqual(t, payload.EncodeString("memo"), req.StartRequest.Memo.Fields["custom"]) + protorequire.ProtoEqual(t, payload.EncodeString("attribute"), req.StartRequest.SearchAttributes.IndexedFields["CustomKeywordField"]) + creations++ + if attempts == 1 && boundary == "after_destination" { + return nil, serviceerror.NewUnavailable("injected after destination commit") + } + return &historyservice.StartWorkflowExecutionResponse{RunId: "destination-run"}, nil + }) + dropped, err := executeRollbackTask(t, e, handler, faultEngine) + require.False(t, dropped) + if boundary == "success" { + require.NoError(t, err) + } else { + require.Error(t, err) + _, err = executeRollbackTask(t, e, handler, faultEngine) + require.NoError(t, err) + } + require.Equal(t, 1, creations) + require.NotNil(t, destination) + require.True(t, destination.Schedule.State.Paused) + require.Equal(t, "user maintenance", destination.Schedule.State.Notes) + require.Len(t, destination.State.BufferedStarts, 1) + require.True(t, destination.State.BufferedStarts[0].Manual) + require.NoError(t, e.readScheduler(func(s *scheduler.Scheduler, _ chasm.Context) error { + require.True(t, s.Closed) + require.Nil(t, s.WorkflowMigration) + return nil + })) + dropped, err = executeRollbackTask(t, e, handler, faultEngine) + require.NoError(t, err) + require.True(t, dropped, "duplicate task must not create another destination") + }) + } +} + +func TestMigrationCounterexample_RollbackDestinationCollision(t *testing.T) { + requireMigrationCounterexamples(t) + e, handler, historyClient := newRollbackScenario(t) + historyClient.EXPECT().StartWorkflowExecution(gomock.Any(), gomock.Any()).Return(nil, + serviceerror.NewWorkflowExecutionAlreadyStarted("unrelated scheduler owns this ID", "unrelated-create-request", "unrelated-run")) + _, err := executeRollbackTask(t, e, handler, e.engine) + require.NoError(t, e.readScheduler(func(s *scheduler.Scheduler, _ chasm.Context) error { + require.False(t, s.Closed, "rollback must retain the source and its acknowledged trigger when a different workflow owns the destination ID") + return nil + })) + require.Error(t, err, "an unrelated destination is a conflict, not a successful handoff") +} diff --git a/chasm/lib/scheduler/migration_rollback_phase_test.go b/chasm/lib/scheduler/migration_rollback_phase_test.go new file mode 100644 index 00000000000..6adc46e86a8 --- /dev/null +++ b/chasm/lib/scheduler/migration_rollback_phase_test.go @@ -0,0 +1,89 @@ +package scheduler_test + +import ( + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + schedulespb "go.temporal.io/server/api/schedule/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/scheduler" + schedulerpb "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" + "go.temporal.io/server/chasm/lib/scheduler/migration" + "go.temporal.io/server/common" + "go.temporal.io/server/common/testing/protorequire" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestRollbackPhaseNativeControl(t *testing.T) { + for _, phase := range []string{"ready", "retry", "cancel", "terminate", "mixed"} { + t.Run(phase, func(t *testing.T) { + s, ctx, _ := setupSchedulerForTest(t) + i := s.Invoker.Get(ctx) + i.InvokerState = rollbackPhaseState(phase) + h := scheduler.NewInvokerExecuteTaskHandler(scheduler.InvokerTaskHandlerOptions{Config: defaultConfig()}) + if phase == "retry" { + i.LastProcessedTime = i.BufferedStarts[0].BackoffTime + } + valid, err := h.Validate(ctx, i, chasm.TaskInvocation{}, &schedulerpb.InvokerExecuteTask{}) + require.NoError(t, err) + require.True(t, valid) + }) + } +} + +func TestRollbackPhaseCounterexample(t *testing.T) { + if os.Getenv("TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES") != "1" { + t.Skip("set TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES=1") + } + testRollbackPhaseGate(t) +} + +func testRollbackPhaseGate(t *testing.T) { + t.Helper() + for _, phase := range []string{"ready", "retry", "cancel", "terminate", "mixed"} { + t.Run(phase, func(t *testing.T) { + s, ctx, _ := setupSchedulerForTest(t) + i := s.Invoker.Get(ctx) + i.InvokerState = rollbackPhaseState(phase) + before := common.CloneProto(i.InvokerState) + _, err := s.MigrateToWorkflow(ctx, &schedulerpb.MigrateToWorkflowRequest{}) + require.Error(t, err, "seed=rollback-%s: V1 cannot execute persisted invoker phase; retain native task ownership", phase) + require.Nil(t, s.WorkflowMigration) + require.False(t, s.Schedule.State.Paused) + protorequire.ProtoEqual(t, before, i.InvokerState) + }) + } +} + +func rollbackPhaseState(phase string) *schedulerpb.InvokerState { + now := timestamppb.New(time.Date(2026, 9, 4, 12, 0, 0, 0, time.UTC)) + i := &schedulerpb.InvokerState{LastProcessedTime: now} + if phase == "ready" || phase == "retry" || phase == "mixed" { + i.BufferedStarts = []*schedulespb.BufferedStart{{RequestId: "pending-request", WorkflowId: "pending-workflow", NominalTime: now, ActualTime: now, DesiredTime: now, Manual: true, Attempt: 1, OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL}} + } + if phase == "retry" { + i.BufferedStarts[0].Attempt = 3 + i.BufferedStarts[0].BackoffTime = timestamppb.New(now.AsTime().Add(time.Hour)) + } + if phase == "cancel" || phase == "mixed" { + i.CancelWorkflows = []*commonpb.WorkflowExecution{{WorkflowId: "cancel-workflow", RunId: "cancel-run"}} + } + if phase == "terminate" || phase == "mixed" { + i.TerminateWorkflows = []*commonpb.WorkflowExecution{{WorkflowId: "terminate-workflow", RunId: "terminate-run"}} + } + return i +} + +func TestRollbackPhaseConversionPreservesFieldsButLosesQueues(t *testing.T) { + s, ctx, _ := setupSchedulerForTest(t) + i := rollbackPhaseState("mixed") + args := migration.CHASMToLegacyStartScheduleArgs(s.SchedulerState, s.Generator.Get(ctx).GeneratorState, i, nil, nil, nil, nil, ctx.Now(s)) + protorequire.ProtoEqual(t, i.BufferedStarts[0], args.State.BufferedStarts[0]) + require.Empty(t, args.Info.RunningWorkflows) + require.Len(t, i.CancelWorkflows, 1) + require.Len(t, i.TerminateWorkflows, 1) +} diff --git a/chasm/lib/scheduler/result_reordering_test.go b/chasm/lib/scheduler/result_reordering_test.go new file mode 100644 index 00000000000..bb90a03b6e2 --- /dev/null +++ b/chasm/lib/scheduler/result_reordering_test.go @@ -0,0 +1,33 @@ +package scheduler_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + enumspb "go.temporal.io/api/enums/v1" + schedulespb "go.temporal.io/server/api/schedule/v1" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestExecuteResultsFollowIdentityAcrossBufferMovement(t *testing.T) { + t.Parallel() + env := newTestEnv(t) + ctx := env.MutableContext() + invoker := env.Scheduler.Invoker.Get(ctx) + now := timestamppb.New(env.TimeSource.Now()) + first := &schedulespb.BufferedStart{RequestId: "first", WorkflowId: "first-workflow", Attempt: 1, OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ALL} + second := &schedulespb.BufferedStart{RequestId: "second", WorkflowId: "second-workflow", Attempt: 1, OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ALL} + invoker.BufferedStarts = []*schedulespb.BufferedStart{second, first} + result := []*schedulespb.BufferedStart{ + {RequestId: "first", RunId: "first-run", StartTime: now}, + {RequestId: "second", RunId: "second-run", StartTime: now}, + } + started, duplicates, _ := invoker.RecordExecuteResult(ctx, result, nil) + require.Equal(t, 2, started) + require.Zero(t, duplicates) + require.Equal(t, "second-run", invoker.BufferedStarts[0].GetRunId()) + require.Equal(t, "first-run", invoker.BufferedStarts[1].GetRunId()) + started, duplicates, _ = invoker.RecordExecuteResult(ctx, result, nil) + require.Zero(t, started) + require.Equal(t, 2, duplicates) +} diff --git a/chasm/lib/scheduler/schedulertest/check_mutations.py b/chasm/lib/scheduler/schedulertest/check_mutations.py new file mode 100644 index 00000000000..9a6cd626e05 --- /dev/null +++ b/chasm/lib/scheduler/schedulertest/check_mutations.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""Run targeted regressions against production mutations using Go overlays.""" +import json +import os +from pathlib import Path +import subprocess +import tempfile + +root = Path(__file__).resolve().parents[4] +mutations = [ + ("completion.go", "early-callback", "./chasm/lib/scheduler/schedulertest", + "TestStartAcceptanceCallbackAndRecording/AllowAll/before", + '\tif !internal.TracksCompletionResult(start.GetOverlapPolicy()) && start.GetRunId() == "" {', + '\tif start.GetCompleted() != nil {'), + ("invoker.go", "deferred-rearm", "./chasm/lib/scheduler/schedulertest", + "TestExpiredSelectionReleasesRemainingBuffer|TestLedger_DeferredStartsStrandedByPausedCompletion", + '\t// A discarded selection cannot produce', '\t// Re-arm tasks if this call changed state'), + ("backfiller_tasks.go", "capacity-progress", "./chasm/lib/scheduler", + "TestBackfillCapacityProgress", None, None), +] +for filename, name, package, test, begin, end in mutations: + source = root / "chasm/lib/scheduler" / filename + text = source.read_text() + if begin is not None: + first = text.index(begin) + last = text.index(end, first) + text = text[:first] + text[last:] + else: + original = "return min(available, max(1, available/backfillerCount))" + assert text.count(original) == 1 + text = text.replace(original, "return available / backfillerCount") + with tempfile.TemporaryDirectory(prefix="scheduler-mutation-") as temp: + directory = Path(temp) + replacement = directory / filename + replacement.write_text(text) + overlay = directory / "overlay.json" + overlay.write_text(json.dumps({"Replace": {str(source): str(replacement)}})) + result = subprocess.run( + ["go", "test", "-tags", "test_dep", "-overlay", str(overlay), package, + "-run", test, "-count=1"], + cwd=root, env={**os.environ, "CGO_ENABLED": "0"}, + text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + ) + if result.returncode == 0 or "--- FAIL: " not in result.stdout or "[build failed]" in result.stdout: + raise SystemExit(f"{name}: mutation did not produce an assertion failure\n{result.stdout}") + print(f"{name}: detected") diff --git a/chasm/lib/scheduler/schedulertest/driver.go b/chasm/lib/scheduler/schedulertest/driver.go index 93e8b94cb89..937b90d3c00 100644 --- a/chasm/lib/scheduler/schedulertest/driver.go +++ b/chasm/lib/scheduler/schedulertest/driver.go @@ -1,10 +1,12 @@ package schedulertest import ( + "cmp" "context" "errors" "fmt" "maps" + "slices" "sync" "testing" "time" @@ -23,6 +25,7 @@ import ( "go.temporal.io/server/chasm/chasmtest" "go.temporal.io/server/chasm/lib/scheduler" schedulerpb "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" + "go.temporal.io/server/common" "go.temporal.io/server/common/clock" "go.temporal.io/server/common/log" "go.temporal.io/server/common/metrics" @@ -52,6 +55,14 @@ var maxKeyFireTime = tasks.MaximumKey.FireTime // re-arms a generator task), so callers bound the run with maxSteps; quiescence is // reached only for paused/exhausted/closed schedules. type Driver struct { + recordCalls chan *RecordCall + external *externalLedger + startCalls chan *StartCall + sideEffectDone chan error + actionsBefore int + recorded map[string]ExternalStart + initialActionCount int64 + T *testing.T Engine *chasmtest.Engine Registry *chasm.Registry @@ -105,12 +116,13 @@ type Driver struct { } type driverConfig struct { - schedule *schedulepb.Schedule - startTime time.Time - afterStep func(d *Driver) - config *scheduler.Config - maxLag time.Duration - maxRun time.Duration + schedule *schedulepb.Schedule + startTime time.Time + afterStep func(d *Driver) + config *scheduler.Config + maxLag time.Duration + maxRun time.Duration + controlledStarts bool } // DriverOption configures a Driver at construction. @@ -212,6 +224,11 @@ func newDriverScaffold(t *testing.T, opts ...DriverOption) (*Driver, *driverConf // ExecuteTask takes (the action-budget invariant). actionRPCs := new(int) startRequests := make(map[string]struct{}) + external := &externalLedger{starts: make(map[string]ExternalStart)} + var startCalls chan *StartCall + if cfg.controlledStarts { + startCalls = make(chan *StartCall) + } rpcMu := &sync.Mutex{} countAction := func() { rpcMu.Lock() @@ -221,12 +238,30 @@ func newDriverScaffold(t *testing.T, opts ...DriverOption) (*Driver, *driverConf // Default happy-path stubs; tests can layer more specific expectations. frontend.EXPECT().StartWorkflowExecution(gomock.Any(), gomock.Any()). - DoAndReturn(func(_ context.Context, req *workflowservice.StartWorkflowExecutionRequest, _ ...grpc.CallOption) (*workflowservice.StartWorkflowExecutionResponse, error) { + DoAndReturn(func(ctx context.Context, req *workflowservice.StartWorkflowExecutionRequest, _ ...grpc.CallOption) (*workflowservice.StartWorkflowExecutionResponse, error) { countAction() rpcMu.Lock() startRequests[req.GetRequestId()] = struct{}{} rpcMu.Unlock() - return &workflowservice.StartWorkflowExecutionResponse{RunId: "run-id"}, nil + if startCalls != nil { + call := &StartCall{Request: common.CloneProto(req), result: make(chan startResponse, 1), ledger: external} + select { + case startCalls <- call: + case <-ctx.Done(): + return nil, ctx.Err() + } + select { + case result := <-call.result: + return result.response, result.err + case <-ctx.Done(): + return nil, ctx.Err() + } + } + accepted, err := external.accept(req) + if err != nil { + return nil, err + } + return &workflowservice.StartWorkflowExecutionResponse{RunId: accepted.RunID}, nil }).AnyTimes() history.EXPECT().TerminateWorkflowExecution(gomock.Any(), gomock.Any()). DoAndReturn(func(context.Context, *historyservice.TerminateWorkflowExecutionRequest, ...grpc.CallOption) (*historyservice.TerminateWorkflowExecutionResponse, error) { @@ -265,6 +300,8 @@ func newDriverScaffold(t *testing.T, opts ...DriverOption) (*Driver, *driverConf engine := chasmtest.NewEngine(t, registry, chasmtest.WithTimeSource(ts)) d := &Driver{ + recordCalls: make(chan *RecordCall), + external: external, startCalls: startCalls, recorded: make(map[string]ExternalStart), T: t, Engine: engine, Registry: registry, @@ -300,6 +337,7 @@ func (d *Driver) startExecution(startFn func(chasm.MutableContext) (*scheduler.S } d.ref = chasm.NewComponentRef[*scheduler.Scheduler](d.execKey) + d.ReadScheduler(func(s *scheduler.Scheduler, _ chasm.Context) { d.initialActionCount = s.Info.GetActionCount() }) } // Now returns the engine's current clock time. @@ -408,6 +446,21 @@ func (d *Driver) pendingSideEffectTasks() []*tasks.ChasmTask { out = append(out, ct) } } + slices.SortStableFunc(out, func(a, b *tasks.ChasmTask) int { + if cmp := a.GetVisibilityTime().Compare(b.GetVisibilityTime()); cmp != 0 { + return cmp + } + if c := cmp.Compare(a.Info.GetTaskVersionedTransition().GetTransitionCount(), b.Info.GetTaskVersionedTransition().GetTransitionCount()); c != 0 { + return c + } + if c := cmp.Compare(a.Info.GetTaskVersionedTransitionOffset(), b.Info.GetTaskVersionedTransitionOffset()); c != 0 { + return c + } + if c := cmp.Compare(a.Info.GetTypeId(), b.Info.GetTypeId()); c != 0 { + return c + } + return slices.Compare(a.Info.GetPath(), b.Info.GetPath()) + }) return out } @@ -453,49 +506,55 @@ func (d *Driver) StepPure() bool { // current time, without moving the clock. It returns false if none is due. func (d *Driver) StepSideEffect() bool { d.T.Helper() + next := d.nextSideEffect() + if next == nil { + return false + } + before := d.actionCount() + err := d.executeSideEffect(next) + d.finishSideEffect(before, err) + return true +} - now := d.Now() - var next *tasks.ChasmTask - for _, ct := range d.pendingSideEffectTasks() { - if ct.GetVisibilityTime().After(now) { +func (d *Driver) nextSideEffect() *tasks.ChasmTask { + for _, task := range d.pendingSideEffectTasks() { + if task.GetVisibilityTime().After(d.Now()) { continue } - if next == nil || ct.GetVisibilityTime().Before(next.GetVisibilityTime()) { - next = ct + d.dispatchedSideEffects[task] = struct{}{} + inTree, valid, err := d.node().ValidateSideEffectTask(context.Background(), task) + if err != nil { + d.T.Fatalf("ValidateSideEffectTask: %v", err) + } + if inTree && valid { + return task } } - if next == nil { - return false - } + return nil +} - d.dispatchedSideEffects[next] = struct{}{} - actionsBefore := d.actionCount() - execErr := d.node().ExecuteSideEffectTask( - chasm.NewEngineContext(context.Background(), d.Engine), - d.execKey, - next, +func (d *Driver) executeSideEffect(task *tasks.ChasmTask) error { + return d.node().ExecuteSideEffectTask( + chasm.NewEngineContext(context.Background(), d.Engine), d.execKey, task, func(chasm.NodeBackend, chasm.Context, chasm.Component) error { return nil }, ) - if execErr != nil { - // A NotFound means the logical task was superseded/removed before - // dispatch (the backend accumulates physical tasks append-only, so a - // stale one can linger). Production drops these; so do we. - if _, ok := errors.AsType[*serviceerror.NotFound](execErr); !ok { - d.T.Fatalf("ExecuteSideEffectTask failed: %v", execErr) +} + +func (d *Driver) finishSideEffect(before int, err error) { + d.T.Helper() + if err != nil { + if _, ok := errors.AsType[*serviceerror.NotFound](err); !ok { + d.T.Fatalf("ExecuteSideEffectTask failed: %v", err) } } - // Track the most actions any single side-effect task (i.e. one - // InvokerExecuteTask) took, for the action-budget invariant. - if taken := d.actionCount() - actionsBefore; taken > d.maxActionsObserved { - d.maxActionsObserved = taken - } + d.maxActionsObserved = max(d.maxActionsObserved, d.actionCount()-before) if err := d.Engine.CloseTransaction(context.Background(), d.ref); err != nil { - d.T.Fatalf("CloseTransaction after side-effect task failed: %v", err) + d.T.Fatalf("CloseTransaction: %v", err) } + d.observeRecordedStarts() if d.AfterStep != nil { d.AfterStep(d) } - return true } // advanceTo moves the clock to t, never backward, first completing any @@ -623,8 +682,19 @@ func (d *Driver) RunToQuiescence(maxSteps int) (steps int, quiescent bool) { // HasPendingTask reports whether any task (pure or side-effect) is currently // scheduled to fire. func (d *Driver) HasPendingTask() bool { - _, have := d.nextFireTime() - return have + if _, have := d.nextPureFireTime(); have { + return true + } + for _, task := range d.pendingSideEffectTasks() { + inTree, valid, err := d.node().ValidateSideEffectTask(context.Background(), task) + if err != nil { + d.T.Fatalf("ValidateSideEffectTask: %v", err) + } + if inTree && valid { + return true + } + } + return false } // Patch applies a SchedulePatch through the production Scheduler.Patch path in a @@ -697,6 +767,14 @@ func (d *Driver) MigrateToWorkflow() error { // Snapshot captures the observable scheduler state at a settled point (after a // step's CloseTransaction). It is the input to invariant checks. type Snapshot struct { + UntransferableActions int + IdleRouteChecked bool + HasValidIdleTask bool + AcceptedStarts map[string]ExternalStart + RecordedStarts map[string]ExternalStart + ActionCount int64 + InitialActionCount int64 + // Now is the engine clock at capture time. Now time.Time // HasPendingTask is true if any task is scheduled to fire. @@ -751,8 +829,12 @@ type Snapshot struct { // Snapshot reads current scheduler state into a Snapshot. func (d *Driver) Snapshot() Snapshot { d.T.Helper() + d.observeRecordedStarts() tweakables := d.Config.Tweakables(Namespace) snap := Snapshot{ + IdleRouteChecked: true, + AcceptedStarts: d.external.snapshot(), RecordedStarts: maps.Clone(d.recorded), + InitialActionCount: d.initialActionCount, Now: d.Now(), HasPendingTask: d.HasPendingTask(), MaxBufferSize: tweakables.MaxBufferSize, @@ -763,6 +845,8 @@ func (d *Driver) Snapshot() Snapshot { DroppedStarts: d.droppedStarts(), } d.ReadScheduler(func(s *scheduler.Scheduler, ctx chasm.Context) { + snap.HasValidIdleTask = d.validIdleTask(s, ctx) + snap.ActionCount = s.Info.GetActionCount() snap.Closed = s.Closed snap.IsSentinel = s.IsSentinel() snap.Paused = s.Schedule.GetState().GetPaused() @@ -780,6 +864,7 @@ func (d *Driver) Snapshot() Snapshot { snap.GeneratorLPT = g.LastProcessedTime.AsTime() } if inv := s.Invoker.Get(ctx); inv != nil { + snap.UntransferableActions = len(inv.GetCancelWorkflows()) + len(inv.GetTerminateWorkflows()) snap.BufferedStartsCount = len(inv.GetBufferedStarts()) snap.addBufferedStarts(inv.GetBufferedStarts()) if inv.LastProcessedTime != nil { @@ -797,6 +882,9 @@ func (snap *Snapshot) addBufferedStarts(starts []*schedulespb.BufferedStart) { for _, start := range starts { switch { case start.GetRunId() == "": + if start.GetAttempt() > 0 { + snap.UntransferableActions++ + } snap.PendingStarts[start.GetRequestId()] = struct{}{} if start.GetAttempt() == -1 { snap.DeferredStarts++ diff --git a/chasm/lib/scheduler/schedulertest/external.go b/chasm/lib/scheduler/schedulertest/external.go new file mode 100644 index 00000000000..8993e35dc7f --- /dev/null +++ b/chasm/lib/scheduler/schedulertest/external.go @@ -0,0 +1,231 @@ +package schedulertest + +import ( + "context" + "fmt" + "maps" + "sync" + + commonpb "go.temporal.io/api/common/v1" + "go.temporal.io/api/workflowservice/v1" + persistencespb "go.temporal.io/server/api/persistence/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/scheduler" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// ExternalStart records acceptance outside the scheduler's transaction. +type ExternalStart struct { + WorkflowID string + RunID string +} + +type externalLedger struct { + mu sync.Mutex + starts map[string]ExternalStart +} + +func (l *externalLedger) accept(req *workflowservice.StartWorkflowExecutionRequest) (ExternalStart, error) { + l.mu.Lock() + defer l.mu.Unlock() + if start, ok := l.starts[req.GetRequestId()]; ok { + if start.WorkflowID != req.GetWorkflowId() { + return ExternalStart{}, fmt.Errorf("request %s changed workflow identity", req.GetRequestId()) + } + return start, nil + } + start := ExternalStart{WorkflowID: req.GetWorkflowId(), RunID: "run-" + req.GetRequestId()} + l.starts[req.GetRequestId()] = start + return start, nil +} + +func (l *externalLedger) snapshot() map[string]ExternalStart { + l.mu.Lock() + defer l.mu.Unlock() + return maps.Clone(l.starts) +} + +// StartCall is an outstanding RPC. Accept and Respond are distinct events so a +// test can deliver a callback or lose the response after external acceptance. +type StartCall struct { + Request *workflowservice.StartWorkflowExecutionRequest + result chan startResponse + ledger *externalLedger +} + +type startResponse struct { + response *workflowservice.StartWorkflowExecutionResponse + err error +} + +func (c *StartCall) Accept() (ExternalStart, error) { return c.ledger.accept(c.Request) } + +func (c *StartCall) Respond(start ExternalStart, err error) { + c.result <- startResponse{response: &workflowservice.StartWorkflowExecutionResponse{RunId: start.RunID}, err: err} +} + +// WithControlledStarts makes start RPCs wait for explicit Accept/Respond events. +func WithControlledStarts() DriverOption { + return func(c *driverConfig) { c.controlledStarts = true } +} + +// BeginSideEffect runs until the handler blocks at a start RPC or finishes. +// FinishSideEffect must be called before starting another task. Engine access +// between these calls is allowed only while all start RPCs are blocked or the +// handler is waiting at NextRecord. +func (d *Driver) BeginSideEffect() bool { + d.T.Helper() + if d.startCalls == nil || d.sideEffectDone != nil { + d.T.Fatal("controlled starts required; only one task may be in flight") + } + next := d.nextSideEffect() + if next == nil { + return false + } + d.actionsBefore = d.actionCount() + done := make(chan error, 1) + d.sideEffectDone = done + ctx, cancel := context.WithCancel(d.T.Context()) + d.T.Cleanup(func() { + cancel() + if d.sideEffectDone == done { + <-done + d.sideEffectDone = nil + } + }) + node := d.node() + go func() { + engine := &recordingEngine{Engine: d.Engine, calls: d.recordCalls} + done <- node.ExecuteSideEffectTask(chasm.NewEngineContext(ctx, engine), d.execKey, next, + func(chasm.NodeBackend, chasm.Context, chasm.Component) error { return nil }) + }() + return true +} + +func (d *Driver) NextStart() *StartCall { + d.T.Helper() + select { + case call := <-d.startCalls: + return call + case err := <-d.sideEffectDone: + d.sideEffectDone = nil + d.T.Fatalf("task finished before expected start: %v", err) + case <-d.T.Context().Done(): + d.T.Fatal("waiting for start: test canceled") + } + return nil +} + +func (d *Driver) FinishSideEffect() { + d.T.Helper() + for { + select { + case call := <-d.recordCalls: + if err := call.Record(); err != nil { + d.T.Fatalf("record start result: %v", err) + } + case err := <-d.sideEffectDone: + d.sideEffectDone = nil + d.finishSideEffect(d.actionsBefore, err) + return + case <-d.T.Context().Done(): + d.T.Fatal("waiting for side effect: test canceled") + } + } +} + +// NextRecord waits until responses have been collected and the handler is ready +// to record them. The caller can deliver other events before calling Record. +func (d *Driver) NextRecord() *RecordCall { + d.T.Helper() + select { + case call := <-d.recordCalls: + return call + case err := <-d.sideEffectDone: + d.sideEffectDone = nil + d.T.Fatalf("task finished before result recording: %v", err) + case <-d.T.Context().Done(): + d.T.Fatal("waiting for recording: test canceled") + } + return nil +} + +type recordingEngine struct { + chasm.Engine + calls chan *RecordCall +} + +// RecordCall separates successful RPC response delivery from local recording. +// It uses the actual handler's update closure; it is not a durable storage commit. +type RecordCall struct { + engine chasm.Engine + ctx context.Context + ref chasm.ComponentRef + update func(chasm.MutableContext, chasm.Component) error + opts []chasm.TransitionOption + result chan recordResponse +} + +type recordResponse struct { + ref []byte + err error +} + +func (e *recordingEngine) UpdateComponent(ctx context.Context, ref chasm.ComponentRef, + update func(chasm.MutableContext, chasm.Component) error, opts ...chasm.TransitionOption) ([]byte, error) { + call := &RecordCall{engine: e.Engine, ctx: ctx, ref: ref, update: update, opts: opts, result: make(chan recordResponse, 1)} + select { + case e.calls <- call: + case <-ctx.Done(): + return nil, ctx.Err() + } + select { + case result := <-call.result: + return result.ref, result.err + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +func (c *RecordCall) Record() error { + ref, err := c.engine.UpdateComponent(c.ctx, c.ref, c.update, c.opts...) + c.result <- recordResponse{ref: ref, err: err} + return err +} + +// DeliverCompletion also supports accepted starts whose RPC results have not +// been recorded locally, and duplicate or obsolete callback request identities. +func (d *Driver) DeliverCompletion(requestID string) error { + _, err := d.Engine.UpdateComponent(chasm.NewEngineContext(context.Background(), d.Engine), d.ref, + func(ctx chasm.MutableContext, c chasm.Component) error { + return c.(*scheduler.Scheduler).HandleNexusCompletion(ctx, &persistencespb.ChasmNexusCompletion{ + RequestId: requestID, CloseTime: timestamppb.New(d.Now()), + Outcome: &persistencespb.ChasmNexusCompletion_Success{Success: &commonpb.Payload{}}, + }) + }) + return err +} + +func (d *Driver) observeRecordedStarts() { + accepted := d.external.snapshot() + d.ReadScheduler(func(s *scheduler.Scheduler, ctx chasm.Context) { + if s.IsSentinel() { + return + } + for _, start := range s.Invoker.Get(ctx).GetBufferedStarts() { + if start.GetRunId() != "" { + if _, ok := accepted[start.GetRequestId()]; ok { + d.recorded[start.GetRequestId()] = ExternalStart{WorkflowID: start.GetWorkflowId(), RunID: start.GetRunId()} + } + } + } + for _, result := range s.Info.GetRecentActions() { + execution := result.GetStartWorkflowResult() + for id, start := range accepted { + if execution.GetWorkflowId() == start.WorkflowID && execution.GetRunId() == start.RunID { + d.recorded[id] = start + } + } + } + }) +} diff --git a/chasm/lib/scheduler/schedulertest/external_test.go b/chasm/lib/scheduler/schedulertest/external_test.go new file mode 100644 index 00000000000..201820a5f58 --- /dev/null +++ b/chasm/lib/scheduler/schedulertest/external_test.go @@ -0,0 +1,137 @@ +package schedulertest_test + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + enumspb "go.temporal.io/api/enums/v1" + "go.temporal.io/api/serviceerror" + "go.temporal.io/server/chasm/lib/scheduler" + "go.temporal.io/server/chasm/lib/scheduler/schedulertest" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestStartAcceptanceCallbackAndRecording(t *testing.T) { + t.Parallel() + for _, policy := range []enumspb.ScheduleOverlapPolicy{ + enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL, + enumspb.SCHEDULE_OVERLAP_POLICY_SKIP, + enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ALL, + } { + for _, position := range []string{"before_response", "before_record", "after_record"} { + t.Run(policy.String()+"/"+position, func(t *testing.T) { + t.Parallel() + start := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) + schedule := oneShotSchedule(start, time.Minute) + schedule.Policies.OverlapPolicy = policy + d := schedulertest.NewDriver(t, schedulertest.WithSchedule(schedule), schedulertest.WithControlledStarts()) + require.True(t, d.StepPure()) + require.True(t, d.BeginSideEffect()) + call := d.NextStart() + accepted, err := call.Accept() + require.NoError(t, err) + before := d.Snapshot() + require.Len(t, before.AcceptedStarts, 1) + require.Empty(t, before.RecordedStarts) + require.Zero(t, before.ActionCount) + if position == "before_response" { + require.NoError(t, d.DeliverCompletion(call.Request.GetRequestId())) + require.NoError(t, d.DeliverCompletion(call.Request.GetRequestId())) + } + call.Respond(accepted, nil) + record := d.NextRecord() + require.Zero(t, d.Snapshot().ActionCount, "a successful response is not yet a recorded start") + if position == "before_record" { + require.NoError(t, d.DeliverCompletion(call.Request.GetRequestId())) + require.NoError(t, d.DeliverCompletion(call.Request.GetRequestId())) + } + require.NoError(t, record.Record()) + d.FinishSideEffect() + if position == "after_record" { + require.NoError(t, d.DeliverCompletion(call.Request.GetRequestId())) + require.NoError(t, d.DeliverCompletion(call.Request.GetRequestId())) + } + cur := d.Snapshot() + require.Empty(t, schedulertest.CheckInvariants(&before, cur)) + require.Equal(t, int64(1), cur.ActionCount) + require.Equal(t, accepted, cur.RecordedStarts[call.Request.GetRequestId()]) + }) + } + } +} + +func TestLostStartResponseDeduplicatesRetry(t *testing.T) { + t.Parallel() + start := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) + schedule := oneShotSchedule(start, time.Minute) + schedule.Policies.OverlapPolicy = enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL + d := schedulertest.NewDriver(t, schedulertest.WithSchedule(schedule), schedulertest.WithControlledStarts()) + require.True(t, d.StepPure()) + require.True(t, d.BeginSideEffect()) + first := d.NextStart() + accepted, err := first.Accept() + require.NoError(t, err) + first.Respond(accepted, serviceerror.NewUnavailable("accepted but response lost")) + d.FinishSideEffect() + pending := d.Snapshot() + require.Len(t, pending.AcceptedStarts, 1) + require.Len(t, pending.PendingStarts, 1) + require.Zero(t, pending.ActionCount) + require.Empty(t, schedulertest.CheckInvariants(nil, pending)) + require.True(t, d.StepPure()) + require.True(t, d.BeginSideEffect()) + retry := d.NextStart() + require.Equal(t, first.Request.GetRequestId(), retry.Request.GetRequestId()) + require.Equal(t, first.Request.GetWorkflowId(), retry.Request.GetWorkflowId()) + again, err := retry.Accept() + require.NoError(t, err) + require.Equal(t, accepted, again) + retry.Respond(again, nil) + d.FinishSideEffect() + cur := d.Snapshot() + require.Len(t, cur.AcceptedStarts, 1) + require.Equal(t, int64(1), cur.ActionCount) + require.Empty(t, schedulertest.CheckInvariants(&pending, cur)) +} + +func TestAcceptedStartAccountingDetector(t *testing.T) { + t.Parallel() + start := schedulertest.ExternalStart{WorkflowID: "workflow", RunID: "run"} + cases := []struct { + name string + snapshot schedulertest.Snapshot + want string + }{ + {"lost after RPC", schedulertest.Snapshot{AcceptedStarts: map[string]schedulertest.ExternalStart{"a": start}}, "accepted-start-lost"}, + {"double counted", schedulertest.Snapshot{AcceptedStarts: map[string]schedulertest.ExternalStart{"a": start}, RecordedStarts: map[string]schedulertest.ExternalStart{"a": start}, ActionCount: 2}, "start-accounting"}, + {"wrong run", schedulertest.Snapshot{AcceptedStarts: map[string]schedulertest.ExternalStart{"a": start}, RecordedStarts: map[string]schedulertest.ExternalStart{"a": {WorkflowID: "workflow", RunID: "wrong"}}, ActionCount: 1}, "start-identity"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + tc.snapshot.HasPendingTask = true + require.Contains(t, violationNames(schedulertest.CheckInvariants(nil, tc.snapshot)), tc.want) + }) + } +} + +func TestExpiredSelectionReleasesRemainingBuffer(t *testing.T) { + t.Parallel() + start := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) + schedule := schedulertest.DefaultSchedule() + schedule.Spec.Interval[0].Interval = durationpb.New(10 * time.Second) + schedule.Spec.EndTime = timestamppb.New(start.Add(31 * time.Second)) + schedule.Policies.CatchupWindow = durationpb.New(time.Minute) + schedule.Policies.OverlapPolicy = enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ALL + d := schedulertest.NewDriver(t, schedulertest.WithSchedule(schedule), + schedulertest.WithMaxRunDuration(5*time.Minute), + schedulertest.WithConfig(configWith(func(tw *scheduler.Tweakables) { tw.IdleTime = 10 * time.Minute })), + schedulertest.WithAfterStep(schedulertest.CheckInvariantsHook(t))) + for range 3 { + require.True(t, d.Step()) + } + require.True(t, d.StepPure()) + require.Empty(t, d.Snapshot().PendingStarts) +} diff --git a/chasm/lib/scheduler/schedulertest/inflight_test.go b/chasm/lib/scheduler/schedulertest/inflight_test.go new file mode 100644 index 00000000000..fcd8208fcb0 --- /dev/null +++ b/chasm/lib/scheduler/schedulertest/inflight_test.go @@ -0,0 +1,80 @@ +package schedulertest + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + enumspb "go.temporal.io/api/enums/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/scheduler" +) + +func TestStartResultSurvivesBufferReordering(t *testing.T) { + t.Parallel() + schedule := DefaultSchedule() + schedule.State.Paused = true + schedule.Policies.OverlapPolicy = enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL + d := NewDriver(t, WithSchedule(schedule), WithControlledStarts()) + require.NoError(t, d.TriggerImmediately()) + require.NoError(t, d.TriggerImmediately()) + require.True(t, d.BeginSideEffect()) + first, second := d.NextStart(), d.NextStart() + acceptedFirst, err := first.Accept() + require.NoError(t, err) + acceptedSecond, err := second.Accept() + require.NoError(t, err) + _, err = d.Engine.UpdateComponent(chasm.NewEngineContext(context.Background(), d.Engine), d.ref, + func(ctx chasm.MutableContext, c chasm.Component) error { + starts := c.(*scheduler.Scheduler).Invoker.Get(ctx).BufferedStarts + starts[0], starts[1] = starts[1], starts[0] + return nil + }) + require.NoError(t, err) + second.Respond(acceptedSecond, nil) + first.Respond(acceptedFirst, nil) + d.FinishSideEffect() + snap := d.Snapshot() + require.Empty(t, CheckInvariants(nil, snap)) + require.Equal(t, int64(2), snap.ActionCount) + require.Equal(t, acceptedFirst, snap.RecordedStarts[first.Request.GetRequestId()]) + require.Equal(t, acceptedSecond, snap.RecordedStarts[second.Request.GetRequestId()]) +} + +// This detector documents the reservation gap from #59/#62 without importing +// their experimental action schema into the lifecycle driver. +func TestSelectedStartReservationCounterexample(t *testing.T) { + t.Parallel() + schedule := DefaultSchedule() + schedule.State.Paused = true + schedule.Policies.OverlapPolicy = enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ALL + d := NewDriver(t, WithSchedule(schedule), WithControlledStarts()) + require.NoError(t, d.TriggerImmediately()) + require.True(t, d.BeginSideEffect()) + call := d.NextStart() + accepted, err := call.Accept() + require.NoError(t, err) + require.NoError(t, d.TriggerImmediately()) + selected := 0 + d.ReadScheduler(func(s *scheduler.Scheduler, ctx chasm.Context) { + for _, start := range s.Invoker.Get(ctx).BufferedStarts { + if start.GetAttempt() > 0 && start.GetCompleted() == nil { + selected++ + } + } + }) + call.Respond(accepted, nil) + d.FinishSideEffect() + require.Equal(t, 2, selected, "known counterexample: two BUFFER_ALL starts selected before the first acknowledgement; replace with <=1 when fixed") +} + +func TestMigrationCannotExemptExecutingObligations(t *testing.T) { + t.Parallel() + schedule := DefaultSchedule() + schedule.State.Paused = true + d := NewDriver(t, WithSchedule(schedule)) + require.NoError(t, d.TriggerImmediately()) + require.NoError(t, d.MigrateToWorkflow()) + violations := CheckInvariants(nil, d.Snapshot()) + require.Contains(t, violations, Violation{Name: "migration-untransferable", Message: "1 execution/retry/cancel/terminate obligations lost native task ownership"}) +} diff --git a/chasm/lib/scheduler/schedulertest/invariants.go b/chasm/lib/scheduler/schedulertest/invariants.go index f65ed68f741..537001b6b8c 100644 --- a/chasm/lib/scheduler/schedulertest/invariants.go +++ b/chasm/lib/scheduler/schedulertest/invariants.go @@ -72,6 +72,10 @@ func CheckInvariants(prev *Snapshot, cur Snapshot) []Violation { } violations = append(violations, checkLedger(prev, cur)...) + violations = append(violations, checkStartAccounting(cur)...) + if cur.IdleRouteChecked && !cur.Closed && !cur.IdleCloseTime.IsZero() && !cur.HasValidIdleTask { + add("idle-route", "advertised idle deadline %s has no valid logical idle task", cur.IdleCloseTime) + } if prev != nil { // (3) Closed is terminal: a closed schedule must never reopen. @@ -132,14 +136,16 @@ func checkLedger(prev *Snapshot, cur Snapshot) []Violation { violations = append(violations, Violation{Name: name, Message: fmt.Sprintf(format, args...)}) } - if cur.Migrating { - // A migration hands the buffer to the V1 workflow. - return nil + if cur.Migrating && cur.UntransferableActions > 0 { + add("migration-untransferable", "%d execution/retry/cancel/terminate obligations lost native task ownership", cur.UntransferableActions) + } + if cur.Migrating && cur.Closed && len(cur.PendingStarts) > 0 { + add("migration-transfer-unverified", "closed source retains %d obligations without a verified destination", len(cur.PendingStarts)) } // Terminal half: a schedule never closes while a start it buffered is // still waiting for InvokerExecuteTask (temporal#13016). - if cur.Closed && len(cur.PendingStarts) > 0 { + if cur.Closed && !cur.Migrating && len(cur.PendingStarts) > 0 { add("ledger-lost-start", "scheduler closed with %d buffered start(s) never sent to StartWorkflowExecution: %v", len(cur.PendingStarts), slices.Sorted(maps.Keys(cur.PendingStarts))) @@ -170,3 +176,25 @@ func checkLedger(prev *Snapshot, cur Snapshot) []Violation { } return violations } + +func checkStartAccounting(cur Snapshot) []Violation { + var violations []Violation + add := func(name, format string, args ...any) { + violations = append(violations, Violation{Name: name, Message: fmt.Sprintf(format, args...)}) + } + if cur.AcceptedStarts != nil { + for id, accepted := range cur.AcceptedStarts { + recorded, ok := cur.RecordedStarts[id] + if ok && recorded != accepted { + add("start-identity", "request %s recorded %+v, accepted %+v", id, recorded, accepted) + } + if _, pending := cur.PendingStarts[id]; !ok && !pending { + add("accepted-start-lost", "accepted request %s has neither a pending obligation nor a recorded result", id) + } + } + if want := cur.InitialActionCount + int64(len(cur.RecordedStarts)); cur.ActionCount != want { + add("start-accounting", "ActionCount=%d, observed recorded starts plus initial count=%d", cur.ActionCount, want) + } + } + return violations +} diff --git a/chasm/lib/scheduler/schedulertest/ledger_test.go b/chasm/lib/scheduler/schedulertest/ledger_test.go index b930f99f84d..397d85b0188 100644 --- a/chasm/lib/scheduler/schedulertest/ledger_test.go +++ b/chasm/lib/scheduler/schedulertest/ledger_test.go @@ -144,7 +144,8 @@ func TestCheckInvariants_Ledger(t *testing.T) { want: []string{"ledger-lost-start"}, }, { - name: "migration takes the buffer", + name: "migration needs destination evidence", + want: []string{"migration-transfer-unverified"}, prev: schedulertest.Snapshot{HasPendingTask: true, PendingStarts: pending("a")}, cur: schedulertest.Snapshot{Closed: true, Migrating: true, PendingStarts: pending("a")}, }, @@ -176,7 +177,6 @@ func uniq(names []string) []string { // re-process the buffer either; the starts (including a manual trigger, which // pause allows) sit until the idle task closes the schedule. func TestLedger_DeferredStartsStrandedByPausedCompletion(t *testing.T) { - t.Skip("known bug: deferred starts stranded after a completion while paused") const idle = time.Hour start := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) diff --git a/chasm/lib/scheduler/schedulertest/planner_properties_test.go b/chasm/lib/scheduler/schedulertest/planner_properties_test.go new file mode 100644 index 00000000000..708687cdef9 --- /dev/null +++ b/chasm/lib/scheduler/schedulertest/planner_properties_test.go @@ -0,0 +1,37 @@ +package schedulertest_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + enumspb "go.temporal.io/api/enums/v1" + "go.temporal.io/server/chasm/lib/scheduler/schedulertest" + "pgregory.net/rapid" +) + +func TestDeferredBufferOneProperty(t *testing.T) { + rapid.Check(t, func(rt *rapid.T) { + schedule := schedulertest.DefaultSchedule() + schedule.State.Paused = true + schedule.Policies.OverlapPolicy = enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ONE + d := schedulertest.NewDriver(t, schedulertest.WithSchedule(schedule)) + require.NoError(t, d.TriggerImmediately()) + require.True(t, d.StepSideEffect()) + n := rapid.IntRange(1, 20).Draw(rt, "manualOccurrences") + for range n { + require.NoError(t, d.TriggerImmediately()) + } + snap := d.Snapshot() + if len(snap.PendingStarts) != 1 || snap.OverlapSkipped != int64(n-1) { + rt.Fatalf("BUFFER_ONE lost conservation: pending=%d skipped=%d occurrences=%d", len(snap.PendingStarts), snap.OverlapSkipped, n) + } + if snap.ActionCount != 1 { + rt.Fatalf("pending work consumed starts: %d", snap.ActionCount) + } + require.True(t, d.CompleteWorkflow()) + require.True(t, d.StepSideEffect()) + if cur := d.Snapshot(); cur.ActionCount != 2 || len(cur.PendingStarts) != 0 { + rt.Fatalf("completion failed to release exactly one start: count=%d pending=%d", cur.ActionCount, len(cur.PendingStarts)) + } + }) +} diff --git a/chasm/lib/scheduler/schedulertest/property_test.go b/chasm/lib/scheduler/schedulertest/property_test.go index 02f1bf53133..91406004c01 100644 --- a/chasm/lib/scheduler/schedulertest/property_test.go +++ b/chasm/lib/scheduler/schedulertest/property_test.go @@ -1,6 +1,7 @@ package schedulertest_test import ( + "errors" "testing" "time" @@ -42,7 +43,7 @@ func TestSchedulerProperties(t *testing.T) { schedulertest.WithMaxSideEffectLag(idle/2), schedulertest.WithMaxRunDuration(idle/2), ) - hook := checkInvariantsIgnoringKnownBugs(rt) + hook := schedulertest.CheckInvariantsHook(rt) d.AfterStep = hook // Check the initial settled state too. @@ -69,11 +70,11 @@ func TestSchedulerProperties(t *testing.T) { case "complete": d.CompleteWorkflow() case "pause": - _ = d.Pause() + checkPatchError(rt, d.Pause()) case "unpause": - _ = d.Unpause() + checkPatchError(rt, d.Unpause()) case "trigger": - _ = d.TriggerImmediately() + checkPatchError(rt, d.TriggerImmediately()) default: rt.Fatalf("unhandled op %q", op) } @@ -133,23 +134,9 @@ func drawSchedule(rt *rapid.T, start time.Time, idle time.Duration) *schedulepb. return s } -// checkInvariantsIgnoringKnownBugs is CheckInvariantsHook minus one known bug -// (TestLedger_DeferredStartsStrandedByPausedCompletion): a lost start where -// every pending start is deferred and nothing is running to re-enable them. -// Delete this once that bug is fixed. -func checkInvariantsIgnoringKnownBugs(rt *rapid.T) func(*schedulertest.Driver) { - var prev *schedulertest.Snapshot - return func(d *schedulertest.Driver) { - rt.Helper() - cur := d.Snapshot() - for _, v := range schedulertest.CheckInvariants(prev, cur) { - if v.Name == "ledger-lost-start" && cur.RunningWorkflows == 0 && - cur.DeferredStarts == len(cur.PendingStarts) { - continue - } - rt.Fatalf("scheduler invariant violation at step (now=%s): %s", - cur.Now.Format("2006-01-02T15:04:05"), v) - } - prev = &cur +func checkPatchError(t *rapid.T, err error) { + t.Helper() + if err != nil && !errors.Is(err, scheduler.ErrClosed) { + t.Fatalf("unexpected patch failure: %v", err) } } diff --git a/chasm/lib/scheduler/schedulertest/task_validity.go b/chasm/lib/scheduler/schedulertest/task_validity.go new file mode 100644 index 00000000000..4982a161770 --- /dev/null +++ b/chasm/lib/scheduler/schedulertest/task_validity.go @@ -0,0 +1,38 @@ +package schedulertest + +import ( + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/scheduler" + schedulerpb "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" + "go.temporal.io/server/common/metrics" + "google.golang.org/protobuf/proto" +) + +func (d *Driver) validIdleTask(s *scheduler.Scheduler, ctx chasm.Context) bool { + id, ok := d.Registry.TaskIDFor(&schedulerpb.SchedulerIdleTask{}) + if !ok { + d.T.Fatal("idle task is not registered") + } + handler := scheduler.NewSchedulerIdleTaskHandler(scheduler.SchedulerIdleTaskHandlerOptions{ + Config: d.Config, MetricsHandler: metrics.NoopMetricsHandler, BaseLogger: d.Logger, + }) + for _, node := range d.node().Snapshot(nil).Nodes { + for _, task := range node.GetMetadata().GetComponentAttributes().GetPureTasks() { + if task.GetTypeId() != id { + continue + } + value := &schedulerpb.SchedulerIdleTask{} + if err := proto.Unmarshal(task.GetData().GetData(), value); err != nil { + d.T.Fatalf("decode idle task: %v", err) + } + valid, err := handler.Validate(ctx, s, chasm.TaskInvocation{TaskAttributes: chasm.TaskAttributes{ScheduledTime: task.GetScheduledTime().AsTime()}}, value) + if err != nil { + d.T.Fatalf("validate idle task: %v", err) + } + if valid { + return true + } + } + } + return false +} diff --git a/chasm/lib/scheduler/schedulertest/task_validity_test.go b/chasm/lib/scheduler/schedulertest/task_validity_test.go new file mode 100644 index 00000000000..232dea8e9e6 --- /dev/null +++ b/chasm/lib/scheduler/schedulertest/task_validity_test.go @@ -0,0 +1,108 @@ +package schedulertest + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + enumspb "go.temporal.io/api/enums/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/scheduler" + schedulerpb "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" + "go.temporal.io/server/common" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestStalePhysicalTaskIsNotProgress(t *testing.T) { + t.Parallel() + schedule := DefaultSchedule() + schedule.State.Paused = true + schedule.Spec.Interval = nil + d := NewDriver(t, WithSchedule(schedule)) + require.NoError(t, d.TriggerImmediately()) + require.NotEmpty(t, d.pendingSideEffectTasks()) + _, err := d.Engine.UpdateComponent(chasm.NewEngineContext(context.Background(), d.Engine), d.ref, + func(ctx chasm.MutableContext, c chasm.Component) error { + c.(*scheduler.Scheduler).Invoker.Get(ctx).BufferedStarts = nil + return nil + }) + require.NoError(t, err) + require.NotEmpty(t, d.pendingSideEffectTasks(), "physical deliveries survive logical invalidation") + require.False(t, d.HasPendingTask()) + require.False(t, d.StepSideEffect()) + require.Empty(t, d.Snapshot().AcceptedStarts) +} + +func TestOldExecuteTaskCanProcessCurrentWork(t *testing.T) { + t.Parallel() + schedule := DefaultSchedule() + schedule.State.Paused = true + schedule.Policies.OverlapPolicy = enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL + d := NewDriver(t, WithSchedule(schedule)) + require.NoError(t, d.TriggerImmediately()) + tasks := d.pendingSideEffectTasks() + require.NotEmpty(t, tasks) + old := tasks[0] + _, err := d.Engine.UpdateComponent(chasm.NewEngineContext(context.Background(), d.Engine), d.ref, + func(ctx chasm.MutableContext, c chasm.Component) error { + inv := c.(*scheduler.Scheduler).Invoker.Get(ctx) + current := common.CloneProto(inv.BufferedStarts[0]) + current.RequestId = "current-request" + current.WorkflowId = "current-workflow" + inv.BufferedStarts = append(inv.BufferedStarts, current) + inv.BufferedStarts[0].RunId = "original-run" + return nil + }) + require.NoError(t, err) + inTree, valid, err := d.node().ValidateSideEffectTask(context.Background(), old) + require.NoError(t, err) + require.True(t, inTree) + require.True(t, valid) + require.True(t, d.StepSideEffect()) + require.Contains(t, d.Snapshot().AcceptedStarts, "current-request") +} + +func TestIdleRouteRequiresMatchingLogicalTask(t *testing.T) { + t.Parallel() + schedule := DefaultSchedule() + schedule.State.LimitedActions = true + d := NewDriver(t, WithSchedule(schedule)) + require.True(t, d.Snapshot().HasValidIdleTask) + _, err := d.Engine.UpdateComponent(chasm.NewEngineContext(context.Background(), d.Engine), d.ref, + func(ctx chasm.MutableContext, c chasm.Component) error { + s := c.(*scheduler.Scheduler) + s.LastEventTime = timestamppb.New(d.Now().Add(time.Hour)) + s.IdleCloseTime = timestamppb.New(s.GetIdleCloseTime().AsTime().Add(time.Hour)) + // Deliberately omit Generator.Generate, which publishes the replacement. + return nil + }) + require.NoError(t, err) + snap := d.Snapshot() + require.False(t, snap.HasValidIdleTask) + violations := CheckInvariants(nil, snap) + require.Contains(t, violations, Violation{Name: "idle-route", Message: "advertised idle deadline " + snap.IdleCloseTime.String() + " has no valid logical idle task"}) +} + +func TestRetryTaskEqualityAndRearming(t *testing.T) { + t.Parallel() + schedule := DefaultSchedule() + schedule.State.Paused = true + d := NewDriver(t, WithSchedule(schedule)) + require.NoError(t, d.TriggerImmediately()) + deadline := d.Now().Add(time.Minute) + _, err := d.Engine.UpdateComponent(chasm.NewEngineContext(context.Background(), d.Engine), d.ref, + func(ctx chasm.MutableContext, c chasm.Component) error { + inv := c.(*scheduler.Scheduler).Invoker.Get(ctx) + inv.BufferedStarts[0].Attempt = 2 + inv.BufferedStarts[0].BackoffTime = timestamppb.New(deadline) + ctx.AddTask(inv, chasm.TaskAttributes{ScheduledTime: deadline}, &schedulerpb.InvokerProcessBufferTask{}) + return nil + }) + require.NoError(t, err) + require.False(t, d.StepSideEffect()) + require.True(t, d.StepPure()) + require.Equal(t, deadline, d.Now()) + require.True(t, d.StepSideEffect()) + require.Len(t, d.Snapshot().AcceptedStarts, 1) +} diff --git a/chasm/lib/scheduler/schedulertest/testdata/rapid/TestSchedulerProperties/TestSchedulerProperties-20261111145706-76678.fail b/chasm/lib/scheduler/schedulertest/testdata/rapid/TestSchedulerProperties/TestSchedulerProperties-20261111145706-76678.fail new file mode 100644 index 00000000000..9780b2dd1c5 --- /dev/null +++ b/chasm/lib/scheduler/schedulertest/testdata/rapid/TestSchedulerProperties/TestSchedulerProperties-20261111145706-76678.fail @@ -0,0 +1,45 @@ +# 2026/11/11 14:57:06.863260 [TestSchedulerProperties] [rapid] draw idleTime: 600000000000 +# 2026/11/11 14:57:06.863265 [TestSchedulerProperties] [rapid] draw intervalSec: 10 +# 2026/11/11 14:57:06.863266 [TestSchedulerProperties] [rapid] draw jitterSec: 0 +# 2026/11/11 14:57:06.863266 [TestSchedulerProperties] [rapid] draw catchupSec: 60 +# 2026/11/11 14:57:06.863266 [TestSchedulerProperties] [rapid] draw overlapPolicy: 3 +# 2026/11/11 14:57:06.863267 [TestSchedulerProperties] [rapid] draw shape: "endTime" +# 2026/11/11 14:57:06.863268 [TestSchedulerProperties] [rapid] draw firesBeforeEnd: 3 +# 2026/11/11 14:57:06.863268 [TestSchedulerProperties] [rapid] draw paused: false +# 2026/11/11 14:57:06.863268 [TestSchedulerProperties] [rapid] draw limited: false +# 2026/11/11 14:57:06.863356 [TestSchedulerProperties] [rapid] draw numOps: 4 +# 2026/11/11 14:57:06.863357 [TestSchedulerProperties] [rapid] draw op: "step" +# 2026/11/11 14:57:06.863449 [TestSchedulerProperties] [rapid] draw op: "step" +# 2026/11/11 14:57:06.863489 [TestSchedulerProperties] [rapid] draw op: "step" +# 2026/11/11 14:57:06.863529 [TestSchedulerProperties] [rapid] draw op: "pure" +# 2026/11/11 14:57:06.863615 [TestSchedulerProperties] scheduler invariant violation at step (now=2020-01-01T00:10:30): [ledger-lost-start] scheduler closed with 1 buffered start(s) never sent to StartWorkflowExecution: [sched-auto-5ff7b19b-8077-5eee-9c97-bd1c5f9f67b6] +# +v0.4.8#7487548259739501542 +0x0 +0x0 +0x0 +0x0 +0x0 +0x0 +0x0 +0x0 +0x38e38e38e38e4 +0x2 +0x0 +0x1 +0x0 +0x38e38e38e38e4 +0x2 +0x0 +0x0 +0x0 +0x38e38e38e38e4 +0x3 +0x0 +0x0 +0x0 +0x0 +0x0 +0x0 +0x38e38e38e38e4 +0x2 diff --git a/chasm/lib/scheduler/spec_processor_properties_test.go b/chasm/lib/scheduler/spec_processor_properties_test.go new file mode 100644 index 00000000000..f93d18b428d --- /dev/null +++ b/chasm/lib/scheduler/spec_processor_properties_test.go @@ -0,0 +1,199 @@ +package scheduler_test + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + enumspb "go.temporal.io/api/enums/v1" + schedulespb "go.temporal.io/server/api/schedule/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/scheduler" + "go.temporal.io/server/common/clock" + "go.uber.org/mock/gomock" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/durationpb" +) + +func TestProcessTimeRange_ManualPaginationPreservesOccurrences(t *testing.T) { + for _, jitter := range []time.Duration{0, 15 * time.Second, 59 * time.Second, 2 * time.Minute} { + for _, pageSize := range []int{1, 2, 5, 20} { + t.Run(fmt.Sprintf("jitter=%s/page=%d", jitter, pageSize), func(t *testing.T) { + env := newTestEnv(t) + ctx := chasm.NewMutableContext(context.Background(), env.Node) + schedule := defaultSchedule() + schedule.Spec.Jitter = durationpb.New(jitter) + sched, err := scheduler.NewScheduler(ctx, namespace, namespaceID, scheduleID, schedule, nil) + require.NoError(t, err) + processor := newTestSpecProcessor(env.Ctrl) + start := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC) + end := start.Add(17*time.Minute + 59*time.Second) + whole, err := processor.ProcessTimeRange(sched, start, end, + enumspb.SCHEDULE_OVERLAP_POLICY_UNSPECIFIED, sched.WorkflowID(), "backfill", true, nil) + require.NoError(t, err) + require.Len(t, whole.BufferedStarts, 17) + for i, action := range whole.BufferedStarts { + nominal := start.Add(time.Duration(i+1) * time.Minute) + require.Equal(t, nominal, action.NominalTime.AsTime()) + require.False(t, action.ActualTime.AsTime().Before(nominal)) + require.False(t, action.ActualTime.AsTime().After(nominal.Add(jitter))) + require.True(t, action.ActualTime.AsTime().Before(nominal.Add(time.Minute)), "jitter must leave the successor enumerable") + } + + var paged []*schedulespb.BufferedStart + cursor := start + for page := 0; ; page++ { + require.LessOrEqual(t, page, len(whole.BufferedStarts), "pagination failed to terminate") + limit := pageSize + part, err := processor.ProcessTimeRange(sched, cursor, end, + enumspb.SCHEDULE_OVERLAP_POLICY_UNSPECIFIED, sched.WorkflowID(), "backfill", true, &limit) + require.NoError(t, err) + require.Zero(t, part.DroppedCount) + require.Equal(t, pageSize-len(part.BufferedStarts), limit) + paged = append(paged, part.BufferedStarts...) + if len(part.BufferedStarts) == 0 { + break + } + require.True(t, part.LastActionTime.After(cursor), "cursor must make progress") + require.Equal(t, part.BufferedStarts[len(part.BufferedStarts)-1].ActualTime.AsTime(), part.LastActionTime) + cursor = part.LastActionTime + } + + require.Len(t, paged, len(whole.BufferedStarts)) + for i, action := range whole.BufferedStarts { + require.True(t, proto.Equal(action, paged[i]), "occurrence %d changed during pagination", i) + } + }) + } + } +} + +func TestProcessTimeRange_AutomatedCapacityAccounting(t *testing.T) { + for _, occurrences := range []int{1, 2, 7, 17} { + for _, capacity := range []int{-1, 0, 1, 3, 20} { + t.Run(fmt.Sprintf("occurrences=%d/capacity=%d", occurrences, capacity), func(t *testing.T) { + env := newTestEnv(t) + ctx := chasm.NewMutableContext(context.Background(), env.Node) + schedule := defaultSchedule() + schedule.Policies.CatchupWindow = durationpb.New(time.Hour) + sched, err := scheduler.NewScheduler(ctx, namespace, namespaceID, scheduleID, schedule, nil) + require.NoError(t, err) + processor := newTestSpecProcessor(env.Ctrl) + start := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC) + end := start.Add(time.Duration(occurrences) * time.Minute) + limit := capacity + result, err := processor.ProcessTimeRange(sched, start, end, + enumspb.SCHEDULE_OVERLAP_POLICY_UNSPECIFIED, sched.WorkflowID(), "", false, &limit) + require.NoError(t, err) + buffered := min(occurrences, max(capacity, 0)) + require.Len(t, result.BufferedStarts, buffered) + require.Equal(t, int64(occurrences-buffered), result.DroppedCount) + require.Equal(t, capacity-buffered, limit) + require.Equal(t, end, result.LastActionTime) + require.Equal(t, end.Add(time.Minute), result.NextWakeupTime) + for i, action := range result.BufferedStarts { + want := start.Add(time.Duration(i+1) * time.Minute) + require.Equal(t, want, action.NominalTime.AsTime()) + require.Equal(t, want, action.ActualTime.AsTime()) + } + + resumed, err := processor.ProcessTimeRange(sched, result.LastActionTime, end.Add(time.Minute), + enumspb.SCHEDULE_OVERLAP_POLICY_UNSPECIFIED, sched.WorkflowID(), "", false, nil) + require.NoError(t, err) + require.Len(t, resumed.BufferedStarts, 1) + require.Equal(t, end.Add(time.Minute), resumed.BufferedStarts[0].ActualTime.AsTime()) + }) + } + } +} + +func TestProcessTimeRange_JitterCursorBoundaries(t *testing.T) { + env := newTestEnv(t) + ctx := chasm.NewMutableContext(context.Background(), env.Node) + schedule := defaultSchedule() + schedule.Spec.Jitter = durationpb.New(59 * time.Second) + sched, err := scheduler.NewScheduler(ctx, namespace, namespaceID, scheduleID, schedule, nil) + require.NoError(t, err) + processor := newTestSpecProcessor(env.Ctrl) + start := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC) + first, err := processor.NextTime(sched, start) + require.NoError(t, err) + require.Equal(t, start.Add(time.Minute), first.Nominal) + require.True(t, first.Next.After(first.Nominal), "fixture must exercise positive jitter") + require.True(t, first.Next.Before(start.Add(2*time.Minute))) + + t.Run("actual end is inclusive", func(t *testing.T) { + result, err := processor.ProcessTimeRange(sched, start, first.Next, + enumspb.SCHEDULE_OVERLAP_POLICY_UNSPECIFIED, sched.WorkflowID(), "backfill", true, nil) + require.NoError(t, err) + require.Len(t, result.BufferedStarts, 1) + require.Equal(t, first.Nominal, result.BufferedStarts[0].NominalTime.AsTime()) + require.Equal(t, first.Next, result.BufferedStarts[0].ActualTime.AsTime()) + require.Equal(t, first.Next, result.LastActionTime) + + resumed, err := processor.ProcessTimeRange(sched, result.LastActionTime, start.Add(3*time.Minute), + enumspb.SCHEDULE_OVERLAP_POLICY_UNSPECIFIED, sched.WorkflowID(), "backfill", true, nil) + require.NoError(t, err) + require.Len(t, resumed.BufferedStarts, 1) + require.Equal(t, start.Add(2*time.Minute), resumed.BufferedStarts[0].NominalTime.AsTime()) + }) + + t.Run("unvisited jittered occurrence remains enumerable", func(t *testing.T) { + boundary := first.Next.Add(-time.Nanosecond) + before, err := processor.ProcessTimeRange(sched, start, boundary, + enumspb.SCHEDULE_OVERLAP_POLICY_UNSPECIFIED, sched.WorkflowID(), "backfill", true, nil) + require.NoError(t, err) + require.Empty(t, before.BufferedStarts) + require.Equal(t, start, before.LastActionTime) + require.Equal(t, first.Next, before.NextWakeupTime) + + resumed, err := processor.ProcessTimeRange(sched, before.LastActionTime, first.Next, + enumspb.SCHEDULE_OVERLAP_POLICY_UNSPECIFIED, sched.WorkflowID(), "backfill", true, nil) + require.NoError(t, err) + require.Len(t, resumed.BufferedStarts, 1) + require.Equal(t, first.Nominal, resumed.BufferedStarts[0].NominalTime.AsTime()) + require.Equal(t, first.Next, resumed.BufferedStarts[0].ActualTime.AsTime()) + }) +} + +func TestGeneratorTask_JitterPendingOccurrenceSurvivesImmediateGeneration(t *testing.T) { + base := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC) + ts := clock.NewEventTimeSource() + ts.Update(base) + schedule := defaultSchedule() + schedule.Spec.Jitter = durationpb.New(59 * time.Second) + e := newSchedulerTestEngine(t, schedule, withEngineTimeSource(ts)) + _, err := e.engine.FirePureTasks(e.rootRef, base) + require.NoError(t, err) + processor := newRealSpecProcessor(gomock.NewController(t), e.logger) + var actual time.Time + require.NoError(t, e.readScheduler(func(s *scheduler.Scheduler, _ chasm.Context) error { + next, err := processor.NextTime(s, base) + if err != nil { + return err + } + require.Equal(t, base.Add(time.Minute), next.Nominal) + require.True(t, next.Next.After(next.Nominal)) + actual = next.Next + return nil + })) + ts.Update(actual.Add(-time.Nanosecond)) + require.NoError(t, e.updateScheduler(func(s *scheduler.Scheduler, ctx chasm.MutableContext) error { + s.Generator.Get(ctx).Generate(ctx) + return nil + })) + _, err = e.engine.FirePureTasks(e.rootRef, ts.Now()) + require.NoError(t, err) + ts.Update(actual) + _, err = e.engine.FirePureTasks(e.rootRef, actual) + require.NoError(t, err) + require.NoError(t, e.readScheduler(func(s *scheduler.Scheduler, ctx chasm.Context) error { + starts := s.Invoker.Get(ctx).BufferedStarts + require.Len(t, starts, 1, "immediate generation must retain the pending jittered occurrence") + require.Equal(t, base.Add(time.Minute), starts[0].NominalTime.AsTime()) + require.Equal(t, actual, starts[0].ActualTime.AsTime()) + return nil + })) +} diff --git a/chasm/lib/scheduler/task_validity_engine_test.go b/chasm/lib/scheduler/task_validity_engine_test.go new file mode 100644 index 00000000000..e5d9ec5ec5f --- /dev/null +++ b/chasm/lib/scheduler/task_validity_engine_test.go @@ -0,0 +1,276 @@ +package scheduler_test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.temporal.io/api/workflowservice/v1" + schedulespb "go.temporal.io/server/api/schedule/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/chasmtest" + "go.temporal.io/server/chasm/lib/scheduler" + "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" + "go.temporal.io/server/common/clock" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/metrics" + "go.uber.org/mock/gomock" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestGeneratorTask_EngineHighWaterMarkInvalidation(t *testing.T) { + env := newSchedulerTestEngine(t, defaultSchedule()) + handler := newGeneratorTaskHandlerForValidity(t, env.logger) + now := env.timeSource.Now() + taskTime := now.Add(time.Minute) + task := &schedulerpb.GeneratorTask{} + + var generator *scheduler.Generator + err := env.updateScheduler( + func(s *scheduler.Scheduler, ctx chasm.MutableContext) error { + generator = s.Generator.Get(ctx) + generator.LastProcessedTime = timestamppb.New(now) + ctx.AddTask(generator, chasm.TaskAttributes{ScheduledTime: taskTime}, task) + return nil + }) + require.NoError(t, err) + + err = env.updateScheduler( + func(s *scheduler.Scheduler, ctx chasm.MutableContext) error { + generator = s.Generator.Get(ctx) + s.Info.OverlapSkipped++ // unrelated scheduler state + valid, err := handler.Validate(ctx, generator, + chasm.TaskInvocation{TaskAttributes: chasm.TaskAttributes{ScheduledTime: taskTime}}, task) + if err != nil { + return err + } + require.True(t, valid) + generator.LastProcessedTime = timestamppb.New(taskTime) + return nil + }) + require.NoError(t, err) + + dropped, err := chasmtest.ExecutePureTask( + context.Background(), env.engine, generator, handler, chasm.TaskAttributes{ScheduledTime: taskTime}, task) + require.NoError(t, err) + require.True(t, dropped) + + dropped, err = chasmtest.ExecutePureTask( + context.Background(), env.engine, generator, handler, chasm.TaskAttributes{}, task) + require.NoError(t, err) + require.False(t, dropped, "immediate Generator tasks ignore the high water mark") +} + +func newGeneratorTaskHandlerForValidity(t *testing.T, logger log.Logger) *scheduler.GeneratorTaskHandler { + t.Helper() + specProcessor := newRealSpecProcessor(gomock.NewController(t), logger) + return scheduler.NewGeneratorTaskHandler(scheduler.GeneratorTaskHandlerOptions{ + Config: defaultConfig(), + MetricsHandler: metrics.NoopMetricsHandler, + BaseLogger: logger, + SpecProcessor: specProcessor, + SpecBuilder: newLegacySpecBuilder(0, 0), + }) +} + +func TestIdleTask_EngineActivityInvalidatesAndRearms(t *testing.T) { + timeSource := clock.NewEventTimeSource() + now := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) + timeSource.Update(now) + schedule := defaultSchedule() + schedule.State.LimitedActions = true + schedule.State.RemainingActions = 0 + env := newSchedulerTestEngine( + t, schedule, withEngineTimeSource(timeSource)) + handler := scheduler.NewSchedulerIdleTaskHandler(scheduler.SchedulerIdleTaskHandlerOptions{ + Config: defaultConfig(), + MetricsHandler: metrics.NoopMetricsHandler, + BaseLogger: env.logger, + }) + + var oldDeadline time.Time + err := env.readScheduler( + func(s *scheduler.Scheduler, _ chasm.Context) error { + oldDeadline = s.GetIdleCloseTime().AsTime() + return nil + }) + require.NoError(t, err) + require.False(t, oldDeadline.IsZero()) + idleTime := oldDeadline.Sub(now) + err = env.updateScheduler( + func(s *scheduler.Scheduler, _ chasm.MutableContext) error { + s.Info.OverlapSkipped++ // unrelated scheduler state + return nil + }) + require.NoError(t, err) + + err = env.readScheduler( + func(s *scheduler.Scheduler, ctx chasm.Context) error { + valid, err := handler.Validate(ctx, s, + chasm.TaskInvocation{TaskAttributes: chasm.TaskAttributes{ScheduledTime: oldDeadline}}, + &schedulerpb.SchedulerIdleTask{IdleTimeTotal: durationpb.New(idleTime)}) + if err != nil { + return err + } + require.True(t, valid) + return nil + }) + require.NoError(t, err) + + timeSource.Update(now.Add(time.Minute)) + _, err = scheduler.NewTestHandler(env.logger).UpdateSchedule(env.engineCtx, &schedulerpb.UpdateScheduleRequest{ + NamespaceId: namespaceID, + FrontendRequest: &workflowservice.UpdateScheduleRequest{ + Namespace: namespace, + ScheduleId: scheduleID, + Schedule: schedule, + }, + }) + require.NoError(t, err) + + var sched *scheduler.Scheduler + var newDeadline time.Time + err = env.readScheduler( + func(s *scheduler.Scheduler, ctx chasm.Context) error { + sched = s + newDeadline = s.GetIdleCloseTime().AsTime() + valid, err := handler.Validate(ctx, s, + chasm.TaskInvocation{TaskAttributes: chasm.TaskAttributes{ScheduledTime: newDeadline}}, + &schedulerpb.SchedulerIdleTask{IdleTimeTotal: durationpb.New(idleTime)}) + if err != nil { + return err + } + require.True(t, valid) + return nil + }) + require.NoError(t, err) + require.True(t, newDeadline.After(oldDeadline)) + + dropped, err := chasmtest.ExecutePureTask( + context.Background(), env.engine, sched, handler, + chasm.TaskAttributes{ScheduledTime: oldDeadline}, + &schedulerpb.SchedulerIdleTask{IdleTimeTotal: durationpb.New(idleTime)}) + require.NoError(t, err) + require.True(t, dropped) +} + +func TestCallbacksTask_EngineCurrentTupleInvalidation(t *testing.T) { + tests := []struct { + name string + mutate func(*schedulespb.BufferedStart) + }{ + {name: "callback recorded", mutate: func(start *schedulespb.BufferedStart) { start.HasCallback = true }}, + {name: "run ID cleared", mutate: func(start *schedulespb.BufferedStart) { start.RunId = "" }}, + {name: "completion recorded", mutate: func(start *schedulespb.BufferedStart) { + start.Completed = &schedulespb.CompletedResult{} + }}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + env := newSchedulerTestEngine(t, defaultSchedule()) + handler := scheduler.NewSchedulerCallbacksTaskHandler(scheduler.SchedulerCallbacksTaskHandlerOptions{ + Config: defaultConfig(), + }) + task := &schedulerpb.SchedulerCallbacksTask{} + + var sched *scheduler.Scheduler + err := env.updateScheduler( + func(s *scheduler.Scheduler, ctx chasm.MutableContext) error { + sched = s + s.Invoker.Get(ctx).BufferedStarts = []*schedulespb.BufferedStart{{ + RequestId: "running-work", + WorkflowId: "running-workflow", + RunId: "running-run", + Attempt: 1, + }} + ctx.AddTask(s, chasm.TaskAttributes{}, task) + return nil + }) + require.NoError(t, err) + + err = env.updateScheduler( + func(s *scheduler.Scheduler, ctx chasm.MutableContext) error { + sched = s + s.Info.OverlapSkipped++ // unrelated scheduler state + valid, err := handler.Validate(ctx, s, chasm.TaskInvocation{}, task) + if err != nil { + return err + } + require.True(t, valid) + test.mutate(s.Invoker.Get(ctx).BufferedStarts[0]) + return nil + }) + require.NoError(t, err) + + dropped, err := chasmtest.ExecuteSideEffectTask( + context.Background(), env.engine, sched, handler, chasm.TaskAttributes{}, task) + require.NoError(t, err) + require.True(t, dropped) + }) + } +} + +func TestMigrationTask_EngineStateInvalidation(t *testing.T) { + tests := []struct { + name string + mutate func(*scheduler.Scheduler) + }{ + { + name: "migration cleared", + mutate: func(s *scheduler.Scheduler) { + s.WorkflowMigration = nil + }, + }, + { + name: "scheduler closed", + mutate: func(s *scheduler.Scheduler) { + s.Closed = true + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + env := newSchedulerTestEngine(t, defaultSchedule()) + handler := scheduler.NewSchedulerMigrateToWorkflowTaskHandler( + scheduler.SchedulerMigrateToWorkflowTaskHandlerOptions{ + Config: defaultConfig(), + MetricsHandler: metrics.NoopMetricsHandler, + BaseLogger: env.logger, + }) + task := &schedulerpb.SchedulerMigrateToWorkflowTask{} + + var sched *scheduler.Scheduler + err := env.updateScheduler( + func(s *scheduler.Scheduler, ctx chasm.MutableContext) error { + sched = s + s.WorkflowMigration = &schedulerpb.WorkflowMigrationState{} + ctx.AddTask(s, chasm.TaskAttributes{}, task) + return nil + }) + require.NoError(t, err) + + err = env.updateScheduler( + func(s *scheduler.Scheduler, ctx chasm.MutableContext) error { + sched = s + s.Info.OverlapSkipped++ // unrelated scheduler state + valid, err := handler.Validate(ctx, s, chasm.TaskInvocation{}, task) + if err != nil { + return err + } + require.True(t, valid) + test.mutate(s) + return nil + }) + require.NoError(t, err) + + dropped, err := chasmtest.ExecuteSideEffectTask( + context.Background(), env.engine, sched, handler, chasm.TaskAttributes{}, task) + require.NoError(t, err) + require.True(t, dropped) + }) + } +} diff --git a/docs/architecture/chasm-schedulertest-coverage-plan.md b/docs/architecture/chasm-schedulertest-coverage-plan.md new file mode 100644 index 00000000000..3a8b337126e --- /dev/null +++ b/docs/architecture/chasm-schedulertest-coverage-plan.md @@ -0,0 +1,116 @@ +# CHASM scheduler lifecycle driver coverage + +## Commit provenance + +Branch: `test/chasm-scheduler-lifecycle`. +Base: `03ecd1d228d2b8d9d433da55785afc9aeeead918`. +David Porter's [temporalio/temporal#13042](https://github.com/temporalio/temporal/pull/13042) +head `33ffb9e001299e3221e107ce781f832b4a616a93` is imported as one signed commit, +`aa47350a376ce3d993bebc069e862d2362bab8c5`. Its tree exactly matches that source +head. Its message records all twelve original commit IDs and preserves David's +authorship. The following signed commit contains the additions described here. + +## Coverage and reference PRs + +| Reference | Coverage on this branch | +| --- | --- | +| [#35](https://github.com/chaptersix/temporal/pull/35) | `task_validity_engine_test.go` ports generator, idle, callback-repair and migration validation cases. `schedulertest/task_validity_test.go` distinguishes stale physical deliveries from valid current work, checks retry equality/rearming, and detects an advertised idle deadline with no valid logical task. | +| [#41](https://github.com/chaptersix/temporal/pull/41) | `TestDeferredBufferOneProperty` adapts the occupancy/accounting property to actual handlers. Existing `TestCompletedHistoryDoesNotConsumeBackfillCapacity`, capacity arithmetic tests and duplicate-result tests remain part of the required package run; no experimental planner refactor is imported. | +| [#51](https://github.com/chaptersix/temporal/pull/51), [#52](https://github.com/chaptersix/temporal/pull/52) | Ports 450/451/1000-range finite-drain tests and the positive-capacity admission fix. The test supplies capacity by consuming each batch; it checks unique identities, bounds and termination, not cross-schedule fairness. | +| [#42](https://github.com/chaptersix/temporal/pull/42), [#43](https://github.com/chaptersix/temporal/pull/43) | Native exclusive-cursor control plus opt-in migrated cursor/epoch and callback-reconciliation counterexamples in `migration_backfill_test.go`. Existing stamped/legacy task tests remain required. The migration schema/progress-bit fix is not imported. | +| [#53](https://github.com/chaptersix/temporal/pull/53), [#55](https://github.com/chaptersix/temporal/pull/55) | Native phase controls, conversion evidence and opt-in rollback phase counterexamples. The driver reports execution/retry/cancel/terminate obligations made unexecutable by migration; it no longer exempts every migrating state from its ledger. The rollback production gate is not imported. | +| [#59](https://github.com/chaptersix/temporal/pull/59), [#62](https://github.com/chaptersix/temporal/pull/62) | Controlled acceptance/response events, buffer reordering during an outstanding RPC, repeated result reconciliation, and a directed selected-start reservation detector. Workflow scenarios are adapted without importing experimental activity APIs or action schemas. | +| [#70](https://github.com/chaptersix/temporal/pull/70) | Duplicate callbacks before/after local start recording for ALLOW_ALL, SKIP and BUFFER_ALL, plus the early ALLOW_ALL callback guard. This branch retains callback attachment; the separate rollout-dependent callback-omission change is not imported. | +| [#44](https://github.com/chaptersix/temporal/pull/44), [#45](https://github.com/chaptersix/temporal/pull/45) | `migration_handoff_test.go` and the V1 SDK workflow scenarios exercise lost responses, source-close boundaries, destination collisions and signal admission. | +| [#56](https://github.com/chaptersix/temporal/pull/56) | `service/frontend/migration_delete_race_test.go` includes the native control and an opt-in held-create/delete counterexample. V1 termination is mocked. | +| [#57](https://github.com/chaptersix/temporal/pull/57) | `migration_chain_counterexample_test.go` adapts owned-descendant, foreign-chain and expired-first-run ownership checks to this baseline. Receipt persistence and the production ownership protocol are not imported. | + +## Driver changes + +- Start calls can be accepted externally, followed by a callback, a lost response, + or successful response delivery. The real handler then records its result. + Tests run engine operations only while all controlled RPCs are blocked. +- External acceptance has stable run identity and survives response loss. Local + observations retain recorded identities separately. Assertions catch accepted + work disappearing, identity mismatches, missing counts and duplicate accounting. +- Side-effect dispatch and progress checks consult `ValidateSideEffectTask`. + Idle-route checks inspect serialized logical tasks and run the production idle + validator. They do not infer progress from stale physical envelopes alone. +- Unexpected patch errors fail the property test. Only the documented closed + schedule rejection is accepted. +- The broad known-bug exemption is removed. Paused completion and catchup expiry + both exposed a discarded selection leaving the remaining buffer deferred. + A narrowly scoped rearming fix now reconsiders deferred work when a discard + leaves no running or in-flight start. Each such immediate pass removes work; + it cannot repeat forever without progress. +- `spec_processor_properties_test.go` supplies independent arithmetic expectations + for interval occurrences, capacity accounting, pagination and jitter boundaries. + It does not establish calendar enumeration or timezone correctness. + +## Reproducible evidence + +All commands run from the repository root. Results recorded 2026-11-11 with +Go `go1.27.3 darwin/arm64` and Rapid `v1.3.0`. + +```sh +CGO_ENABLED=0 go test -tags test_dep ./chasm/chasmtest/... ./chasm/lib/scheduler/... ./service/frontend ./service/worker/scheduler -count=1 +CGO_ENABLED=0 go test -tags test_dep ./chasm/lib/scheduler/schedulertest -run TestSchedulerProperties -rapid.checks=1000 -count=1 +python3 chasm/lib/scheduler/schedulertest/check_mutations.py +CGO_ENABLED=0 make lint-code-fast GOLANGCI_LINT_BASE_REV=03ecd1d228d2b8d9d433da55785afc9aeeead918 GOLANGCI_LINT_FIX=false +``` + +The package tests, 1,000 Rapid cases, all three mutation checks and fast lint pass. Mutations +remove the early callback guard, deferred rearming, and positive-capacity +admission independently; each produces assertion failures in its targeted tests. +The original early callback run failed on `accepted-start-lost`, with ActionCount +still zero after external acceptance. The fix preserves exactly one counted start. + +The catchup-expiry failure is retained both as +`TestExpiredSelectionReleasesRemainingBuffer` and a Rapid failfile under +`schedulertest/testdata/rapid`. The explicit trace is: fixed initial clock +2020-01-01 UTC; 10-second interval; end after 31 seconds; BUFFER_ALL; 60-second +catchup; 10-minute idle; 5-minute workflow-duration bound; `step, step, step, pure`. +The seeded replay is: + +```sh +CGO_ENABLED=0 go test -tags test_dep ./chasm/lib/scheduler/schedulertest -run TestSchedulerProperties -rapid.seed=7487548259739501542 -count=1 +``` + +## Unresolved counterexamples and limits + +The migration cases are evidence, not fixes. Enable their intended correctness +assertions explicitly; this command is expected to fail on this branch: + +```sh +TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES=1 CGO_ENABLED=0 go test -tags test_dep ./chasm/lib/scheduler ./service/frontend ./service/worker/scheduler -run 'TestMigrationCounterexample|TestMigrationBackfill_(Migrated|Waits)|TestMigrationDeleteCounterexample|TestRollbackPhaseCounterexample' -count=1 +``` + +Observed failures include non-epoch migrated cursor replay, callback-barrier +ordering, rollback execution phases, foreign ownership, source signals during +handoff, ambiguous committed create responses, fresh rollback range boundaries, +delete/create ordering, and descendant ownership verification. The epoch case is +included but does not fail in this fixture; that is not a proof of epoch-state +representation. None of these tests establishes namespace failover safety. + +`TestSelectedStartReservationCounterexample` is an always-run detector asserting +the observed two selected BUFFER_ALL starts before the first acknowledgement. +It records the remaining reservation gap; replace that characterization with a +one-slot correctness assertion when the production fix is ported. + +The default driver remains a bounded, single-schedule environment. Request IDs +are unique per generated occurrence. The external ledger retains deduplication +identities for the whole scenario, assuming no retention expiry. Recorded-result +observation uses buffered starts and recent history: keep each ALLOW_ALL batch +within the retained-history window and observe after every task. Large-batch +history eviction needs a separate observer. Long-running capture/memory work +from [#68](https://github.com/chaptersix/temporal/pull/68) and +[#69](https://github.com/chaptersix/temporal/pull/69) remains a scaling follow-up. + +Serialized restart, physical timer redelivery, durable commits, and History +admission fencing remain separate integration coverage. No new in-memory helper +is presented as persistence or crash evidence. The source-close fault wrapper +only tests transaction outcome/response ordering in the component engine. + +A race-enabled test build was attempted but blocked by the host SDK/linker: +`Security.tbd: unknown architecture arm64e.x1`. This is not a successful race run. +Normal tests use `CGO_ENABLED=0`; rerun the race check on a working C toolchain. diff --git a/service/frontend/migration_delete_race_test.go b/service/frontend/migration_delete_race_test.go new file mode 100644 index 00000000000..0c0f617df0c --- /dev/null +++ b/service/frontend/migration_delete_race_test.go @@ -0,0 +1,103 @@ +package frontend + +import ( + "context" + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" + schedulepb "go.temporal.io/api/schedule/v1" + "go.temporal.io/api/serviceerror" + "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/server/api/historyservice/v1" + "go.temporal.io/server/api/historyservicemock/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/chasmtest" + chasmscheduler "go.temporal.io/server/chasm/lib/scheduler" + schedulerpb "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" + "go.temporal.io/server/common/clock" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/namespace" + legacyscheduler "go.temporal.io/server/service/worker/scheduler" + "go.uber.org/mock/gomock" + "google.golang.org/grpc" + "google.golang.org/protobuf/types/known/timestamppb" +) + +type migrationDeleteRaceClient struct { + schedulerpb.SchedulerServiceClient + engineCtx context.Context + ref chasm.ComponentRef +} + +func (c *migrationDeleteRaceClient) DeleteSchedule(_ context.Context, req *schedulerpb.DeleteScheduleRequest, _ ...grpc.CallOption) (*schedulerpb.DeleteScheduleResponse, error) { + response, _, err := chasm.UpdateComponent(c.engineCtx, c.ref, func(s *chasmscheduler.Scheduler, ctx chasm.MutableContext, _ struct{}) (*schedulerpb.DeleteScheduleResponse, error) { + return s.Delete(ctx, req) + }, struct{}{}) + return response, err +} + +func TestMigrationDeleteNativeControl(t *testing.T) { testMigrationDeleteRace(t, false) } +func TestMigrationDeleteCounterexample(t *testing.T) { + if os.Getenv("TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES") != "1" { + t.Skip("set TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES=1") + } + testMigrationDeleteRace(t, true) +} + +func testMigrationDeleteRace(t *testing.T, inFlight bool) { + t.Helper() + logger := log.NewNoopLogger() + config := &chasmscheduler.Config{Tweakables: func(string) chasmscheduler.Tweakables { return chasmscheduler.DefaultTweakables }} + builder := legacyscheduler.NewSpecBuilder(func() int { return 0 }, func() int { return 0 }) + processor := chasmscheduler.NewSpecProcessor(config, metrics.NoopMetricsHandler, logger, builder) + registry := chasm.NewRegistry(logger) + require.NoError(t, registry.Register(&chasm.CoreLibrary{})) + require.NoError(t, registry.Register(chasmscheduler.NewLibrary(config, nil, + chasmscheduler.NewSchedulerIdleTaskHandler(chasmscheduler.SchedulerIdleTaskHandlerOptions{Config: config, MetricsHandler: metrics.NoopMetricsHandler, BaseLogger: logger}), nil, + chasmscheduler.NewGeneratorTaskHandler(chasmscheduler.GeneratorTaskHandlerOptions{Config: config, MetricsHandler: metrics.NoopMetricsHandler, BaseLogger: logger, SpecBuilder: builder, SpecProcessor: processor}), nil, nil, nil, nil))) + now := time.Date(2026, 9, 4, 12, 0, 0, 0, time.UTC) + engine := chasmtest.NewEngine(t, registry, chasmtest.WithTimeSource(clock.NewEventTimeSource().Update(now))) + engineCtx := chasm.NewEngineContext(context.Background(), engine) + key := chasm.ExecutionKey{NamespaceID: "ns-id", BusinessID: "schedule-id"} + ref := chasm.NewComponentRef[*chasmscheduler.Scheduler](key) + create := func() error { + _, err := chasm.StartExecution(engineCtx, key, chasmscheduler.CreateSchedulerFromMigration, &schedulerpb.CreateFromMigrationStateRequest{NamespaceId: key.NamespaceID, State: &schedulerpb.SchedulerMigrationState{ + SchedulerState: &schedulerpb.SchedulerState{Namespace: "ns", NamespaceId: key.NamespaceID, ScheduleId: key.BusinessID, Schedule: &schedulepb.Schedule{Spec: &schedulepb.ScheduleSpec{}, State: &schedulepb.ScheduleState{Paused: true}}, Info: &schedulepb.ScheduleInfo{}}, + GeneratorState: &schedulerpb.GeneratorState{LastProcessedTime: timestamppb.New(now)}, InvokerState: &schedulerpb.InvokerState{}, + }}) + return err + } + started, release, created := make(chan struct{}), make(chan struct{}), make(chan error, 1) + if inFlight { + go func() { close(started); <-release; created <- create() }() + <-started + } else { + require.NoError(t, create()) + } + ctrl := gomock.NewController(t) + namespaces := namespace.NewMockRegistry(ctrl) + namespaces.EXPECT().GetNamespaceID(namespace.Name("ns")).Return(namespace.ID("ns-id"), nil).Times(2) + history := historyservicemock.NewMockHistoryServiceClient(ctrl) + history.EXPECT().TerminateWorkflowExecution(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, req *historyservice.TerminateWorkflowExecutionRequest, _ ...grpc.CallOption) (*historyservice.TerminateWorkflowExecutionResponse, error) { + require.Equal(t, legacyscheduler.WorkflowIDPrefix+key.BusinessID, req.TerminateRequest.WorkflowExecution.WorkflowId) + return &historyservice.TerminateWorkflowExecutionResponse{}, nil + }).Times(1) + handler := &WorkflowHandler{logger: logger, namespaceRegistry: namespaces, historyClient: history, schedulerClient: &migrationDeleteRaceClient{engineCtx: engineCtx, ref: ref}, config: &Config{EnableSchedules: func(string) bool { return true }, EnableCHASMSchedulerCreation: func(string) bool { return true }}} + _, deleteErr := handler.DeleteSchedule(context.Background(), &workflowservice.DeleteScheduleRequest{Namespace: "ns", ScheduleId: key.BusinessID}) + if inFlight { + close(release) + require.NoError(t, <-created) + } + require.NoError(t, deleteErr) + _, err := chasm.ReadComponent(engineCtx, ref, func(s *chasmscheduler.Scheduler, _ chasm.Context, _ struct{}) (struct{}, error) { + require.True(t, s.Closed, "seed=delete-create: successful DeleteSchedule must not be followed by an active migrated scheduler") + return struct{}{}, nil + }, struct{}{}) + if err != nil { + var notFound *serviceerror.NotFound + require.ErrorAs(t, err, ¬Found) + } +} diff --git a/service/worker/scheduler/migration_scenarios_test.go b/service/worker/scheduler/migration_scenarios_test.go new file mode 100644 index 00000000000..9f6c9f2561d --- /dev/null +++ b/service/worker/scheduler/migration_scenarios_test.go @@ -0,0 +1,271 @@ +package scheduler + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + schedulepb "go.temporal.io/api/schedule/v1" + "go.temporal.io/api/serviceerror" + taskqueuepb "go.temporal.io/api/taskqueue/v1" + workflowpb "go.temporal.io/api/workflow/v1" + "go.temporal.io/sdk/testsuite" + "go.temporal.io/sdk/workflow" + schedulespb "go.temporal.io/server/api/schedule/v1" + schedulerpb "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" + "go.temporal.io/server/chasm/lib/scheduler/migration" + "go.temporal.io/server/common" + "google.golang.org/grpc" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func newMigrationScenario(t *testing.T) (*testsuite.TestWorkflowEnvironment, *schedulespb.StartScheduleArgs) { + t.Helper() + var suite testsuite.WorkflowTestSuite + env := suite.NewTestWorkflowEnvironment() + env.SetStartTime(baseStartTime) + env.RegisterDelayedCallback(func() { env.SignalWorkflow(SignalNameForceCAN, nil) }, 3*time.Hour) + t.Cleanup(func() { env.AssertExpectations(t) }) + return env, &schedulespb.StartScheduleArgs{ + Schedule: &schedulepb.Schedule{ + Spec: &schedulepb.ScheduleSpec{Interval: []*schedulepb.IntervalSpec{{Interval: durationpb.New(time.Hour)}}}, + Action: &schedulepb.ScheduleAction{Action: &schedulepb.ScheduleAction_StartWorkflow{ + StartWorkflow: &workflowpb.NewWorkflowExecutionInfo{ + WorkflowId: "action", WorkflowType: &commonpb.WorkflowType{Name: "action"}, TaskQueue: &taskqueuepb.TaskQueue{Name: "queue"}, + }, + }}, + Policies: &schedulepb.SchedulePolicies{OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL}, + State: &schedulepb.ScheduleState{}, + }, + Info: &schedulepb.ScheduleInfo{}, + State: &schedulespb.InternalState{Namespace: "ns", NamespaceId: testNamespaceID, ScheduleId: "schedule", ConflictToken: InitialConflictToken}, + } +} + +func requireMigrationCounterexamples(t *testing.T) { + t.Helper() + if os.Getenv("TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES") == "" { + t.Skip("set TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES=1 to run known failing migration repros") + } +} + +func TestMigrationScenario_TriggerBeforeSnapshot(t *testing.T) { + checkMigrationSignalAccounting(t, false) +} + +func TestMigrationCounterexample_TriggerDuringSnapshot(t *testing.T) { + requireMigrationCounterexamples(t) + checkMigrationSignalAccounting(t, true) +} + +func checkMigrationSignalAccounting(t *testing.T, duringMigration bool) { + t.Helper() + env, args := newMigrationScenario(t) + enabled := duringMigration + var destination *schedulerpb.SchedulerMigrationState + starts := 0 + env.OnActivity(new(activities).StartWorkflow, mock.Anything, mock.Anything).Maybe().Return( + func(context.Context, *schedulespb.StartWorkflowRequest) (*schedulespb.StartWorkflowResponse, error) { + starts++ + return &schedulespb.StartWorkflowResponse{RunId: "run", RealStartTime: timestamppb.New(env.Now())}, nil + }) + env.OnActivity(new(activities).MigrateScheduleToChasm, mock.Anything, mock.Anything).Once().After(2 * time.Second).Return( + func(_ context.Context, req *schedulerpb.CreateFromMigrationStateRequest) error { + destination = common.CloneProto(req.State) + return nil + }) + delivered := false + env.RegisterDelayedCallback(func() { + require.False(t, env.IsWorkflowCompleted()) + env.SignalWorkflow(SignalNamePatch, &schedulepb.SchedulePatch{ + TriggerImmediately: &schedulepb.TriggerImmediatelyRequest{OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL}, + }) + delivered = true + }, time.Second) + if !duringMigration { + env.RegisterDelayedCallback(func() { + enabled = true + env.SignalWorkflow(SignalNameMigrateToChasm, nil) + }, 3*time.Second) + } + env.ExecuteWorkflow(func(ctx workflow.Context, input *schedulespb.StartScheduleArgs) error { + return schedulerWorkflowWithSpecBuilder(ctx, input, newSpecBuilderForTest(0, 0), schedulerDynamicConfig{ + enableCHASMMigration: func() bool { return enabled }, migrateWithRunningWorkflows: func() bool { return true }, + versionOverride: func() int { return int(MigrationHandoffFixes) }, + }) + }, args) + require.NoError(t, env.GetWorkflowError()) + require.True(t, delivered) + require.NotNil(t, destination) + pending := 0 + for _, start := range destination.InvokerState.BufferedStarts { + if start.Manual && start.RunId == "" && start.Completed == nil { + pending++ + } + } + for _, backfill := range destination.Backfillers { + if backfill.GetTriggerRequest() != nil { + pending++ + } + } + require.Equal(t, 1, starts+pending, "a delivered trigger must execute or remain actionable at the destination") +} + +type migrationCommitClient struct { + schedulerpb.SchedulerServiceClient + destination *schedulerpb.SchedulerMigrationState + commit bool + calls int +} + +func (c *migrationCommitClient) CreateFromMigrationState( + _ context.Context, req *schedulerpb.CreateFromMigrationStateRequest, _ ...grpc.CallOption, +) (*schedulerpb.CreateFromMigrationStateResponse, error) { + c.calls++ + if c.calls == 1 { + if c.commit { + c.destination = common.CloneProto(req.State) + } + return nil, serviceerror.NewUnavailable("injected response loss") + } + if c.destination != nil { + return nil, serviceerror.NewAlreadyExists("destination committed on first attempt") + } + c.destination = common.CloneProto(req.State) + return &schedulerpb.CreateFromMigrationStateResponse{}, nil +} + +func TestMigrationScenario_FailureBeforeCommit(t *testing.T) { + checkMigrationCommitAccounting(t, false, false, MigrationHandoffFixes) +} + +func TestMigrationScenario_NativeBackfillIncludesStart(t *testing.T) { + checkRollbackBackfillStart(t, false) +} + +func TestMigrationCounterexample_RollbackFreshBackfill(t *testing.T) { + requireMigrationCounterexamples(t) + checkRollbackBackfillStart(t, true) +} + +func TestMigrationCounterexample_ForwardDestinationCollision(t *testing.T) { + requireMigrationCounterexamples(t) + env, args := newMigrationScenario(t) + when := timestamppb.New(baseStartTime) + args.State.BufferedStarts = []*schedulespb.BufferedStart{{ + NominalTime: when, ActualTime: when, Manual: true, + OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL, + }} + client := &mockSchedulerClient{migrateErr: serviceerror.NewAlreadyExists("unrelated CHASM schedule owns this ID")} + a := newTestActivities(client, testNamespaceID) + env.OnActivity(new(activities).MigrateScheduleToChasm, mock.Anything, mock.Anything).Return( + func(ctx context.Context, req *schedulerpb.CreateFromMigrationStateRequest) error { + return a.MigrateScheduleToChasm(ctx, req) + }) + env.ExecuteWorkflow(func(ctx workflow.Context, input *schedulespb.StartScheduleArgs) error { + return schedulerWorkflowWithSpecBuilder(ctx, input, newSpecBuilderForTest(0, 0), schedulerDynamicConfig{ + enableCHASMMigration: func() bool { return true }, migrateWithRunningWorkflows: func() bool { return true }, + }) + }, args) + require.True(t, workflow.IsContinueAsNewError(env.GetWorkflowError()), + "the source must remain active when the destination ID belongs to an unrelated schedule") +} + +func checkRollbackBackfillStart(t *testing.T, rollback bool) { + t.Helper() + env, args := newMigrationScenario(t) + args.Schedule.State.Paused = true + env.RegisterDelayedCallback(func() { env.SignalWorkflow(SignalNameForceCAN, nil) }, time.Second) + when := timestamppb.New(baseStartTime.Add(-time.Hour)) + backfill := &schedulepb.BackfillRequest{ + StartTime: when, EndTime: when, OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL, + } + if rollback { + args = migration.CHASMToLegacyStartScheduleArgs(&schedulerpb.SchedulerState{ + Schedule: args.Schedule, Info: args.Info, Namespace: args.State.Namespace, + NamespaceId: args.State.NamespaceId, ScheduleId: args.State.ScheduleId, + }, &schedulerpb.GeneratorState{LastProcessedTime: timestamppb.New(baseStartTime)}, &schedulerpb.InvokerState{}, + map[string]*schedulerpb.BackfillerState{"fresh": { + Request: &schedulerpb.BackfillerState_BackfillRequest{BackfillRequest: backfill}, + }}, nil, nil, nil, baseStartTime) + } else { + args.InitialPatch = &schedulepb.SchedulePatch{BackfillRequest: []*schedulepb.BackfillRequest{backfill}} + } + starts := 0 + env.OnActivity(new(activities).StartWorkflow, mock.Anything, mock.Anything).Maybe().Return( + func(context.Context, *schedulespb.StartWorkflowRequest) (*schedulespb.StartWorkflowResponse, error) { + starts++ + return &schedulespb.StartWorkflowResponse{RunId: "backfilled-run", RealStartTime: timestamppb.New(env.Now())}, nil + }) + env.ExecuteWorkflow(SchedulerWorkflow, args) + require.True(t, workflow.IsContinueAsNewError(env.GetWorkflowError())) + require.Equal(t, 1, starts, "the inclusive start of a fresh CHASM backfill must remain actionable after rollback") +} + +func TestMigrationCounterexample_LostCreateResponse(t *testing.T) { + requireMigrationCounterexamples(t) + for _, version := range []SchedulerWorkflowVersion{TriggerImmediatelyTimestamp, MigrationHandoffFixes} { + for _, disable := range []bool{false, true} { + t.Run(fmt.Sprintf("version_%d_disable_%t", version, disable), func(t *testing.T) { + checkMigrationCommitAccounting(t, true, disable, version) + }) + } + } +} + +func checkMigrationCommitAccounting(t *testing.T, commit, disable bool, version SchedulerWorkflowVersion) { + t.Helper() + env, args := newMigrationScenario(t) + when := timestamppb.New(baseStartTime) + args.State.BufferedStarts = []*schedulespb.BufferedStart{{ + NominalTime: when, ActualTime: when, Manual: true, OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL, + }} + client := &migrationCommitClient{commit: commit} + a := newTestActivities(client, testNamespaceID) + enabled := true + a.migrationEnabled = func() bool { return enabled } + env.OnActivity(new(activities).MigrateScheduleToChasm, mock.Anything, mock.Anything).Return( + func(ctx context.Context, req *schedulerpb.CreateFromMigrationStateRequest) error { + err := a.MigrateScheduleToChasm(ctx, req) + if disable { + enabled = false + } + return err + }) + var sourceRequests []*schedulespb.StartWorkflowRequest + env.OnActivity(new(activities).StartWorkflow, mock.Anything, mock.Anything).Return( + func(_ context.Context, req *schedulespb.StartWorkflowRequest) (*schedulespb.StartWorkflowResponse, error) { + sourceRequests = append(sourceRequests, common.CloneProto(req)) + return &schedulespb.StartWorkflowResponse{RunId: fmt.Sprintf("run-%d", len(sourceRequests)), RealStartTime: timestamppb.New(env.Now())}, nil + }) + env.ExecuteWorkflow(func(ctx workflow.Context, input *schedulespb.StartScheduleArgs) error { + return schedulerWorkflowWithSpecBuilder(ctx, input, newSpecBuilderForTest(0, 0), schedulerDynamicConfig{ + enableCHASMMigration: func() bool { return enabled }, migrateWithRunningWorkflows: func() bool { return true }, + versionOverride: func() int { return int(version) }, versionCeiling: func() int { return int(version) }, + }) + }, args) + if disable { + require.True(t, workflow.IsContinueAsNewError(env.GetWorkflowError())) + } else { + require.NoError(t, env.GetWorkflowError()) + require.Equal(t, 2, client.calls) + } + require.NotEmpty(t, sourceRequests) + require.NotNil(t, client.destination) + manualAtDestination := 0 + for _, start := range client.destination.InvokerState.BufferedStarts { + if start.Manual && start.RunId == "" && start.Completed == nil { + manualAtDestination++ + t.Logf("destination workflow/request=%s/%s; source workflow/request=%s/%s", start.WorkflowId, start.RequestId, + sourceRequests[0].Request.WorkflowId, sourceRequests[0].Request.RequestId) + } + } + require.Zero(t, manualAtDestination, "source executed the manual action after migration returned an error; the committed destination must not still own it") +} From 410af8fb88d290662849073a854ecd6e5844c7dd Mon Sep 17 00:00:00 2001 From: chaptersix <13949480+chaptersix@users.noreply.github.com> Date: Wed, 11 Nov 2026 16:18:53 -0600 Subject: [PATCH 03/22] Tighten scheduler lifecycle test gates and harness failure modes Require TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES=1 exactly in every gated test and share one skip message. Fail fast when a controlled start RPC is left unconsumed or answered twice instead of blocking until the test timeout. Release the held create in the delete race test on early failure. Assert the rollback conversion output rather than its input fixture, echo the request ID actually sent in the rollback retry mock, and drop misleading seed labels, a debug log, and wall-clock fixtures. Co-Authored-By: Claude Fable 5.2 --- .../scheduler/backfill_capacity_progress_test.go | 2 +- chasm/lib/scheduler/migration_backfill_test.go | 5 +---- chasm/lib/scheduler/migration_handoff_test.go | 10 +++++++--- .../scheduler/migration_rollback_phase_test.go | 11 ++++------- chasm/lib/scheduler/schedulertest/driver.go | 2 +- chasm/lib/scheduler/schedulertest/external.go | 10 +++++++++- .../scheduler/schedulertest/invariants_test.go | 4 ++-- service/frontend/migration_delete_race_test.go | 16 +++++++++++----- .../worker/scheduler/migration_scenarios_test.go | 10 ++++------ 9 files changed, 40 insertions(+), 30 deletions(-) diff --git a/chasm/lib/scheduler/backfill_capacity_progress_test.go b/chasm/lib/scheduler/backfill_capacity_progress_test.go index 8a8c5b2e8e4..4b986005a1f 100644 --- a/chasm/lib/scheduler/backfill_capacity_progress_test.go +++ b/chasm/lib/scheduler/backfill_capacity_progress_test.go @@ -59,7 +59,7 @@ func testBackfillCapacityProgress(t *testing.T, count int) { if remaining == 0 { break } - require.Positive(t, buffered, "seed=capacity-%d: empty buffer and %d ranges must make progress", count, remaining) + require.Positive(t, buffered, "count=%d: empty buffer and %d ranges must make progress", count, remaining) ts.Update(ts.Now().Add(time.Hour)) _, err := e.engine.FirePureTasks(e.rootRef, ts.Now()) require.NoError(t, err) diff --git a/chasm/lib/scheduler/migration_backfill_test.go b/chasm/lib/scheduler/migration_backfill_test.go index e00c36741b1..fb175bea16c 100644 --- a/chasm/lib/scheduler/migration_backfill_test.go +++ b/chasm/lib/scheduler/migration_backfill_test.go @@ -1,7 +1,6 @@ package scheduler_test import ( - "os" "testing" "time" @@ -30,9 +29,7 @@ func TestMigrationBackfill_NativeResumesAfterWatermark(t *testing.T) { } func TestMigrationBackfill_MigratedResumesAfterWatermark(t *testing.T) { - if os.Getenv("TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES") == "" { - t.Skip("set TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES=1 to run known failing migration repros") - } + requireMigrationCounterexamples(t) for _, end := range []time.Time{time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC), time.Unix(60, 0).UTC()} { t.Run(end.String(), func(t *testing.T) { checkMigrationBackfillWatermark(t, true, end) }) } diff --git a/chasm/lib/scheduler/migration_handoff_test.go b/chasm/lib/scheduler/migration_handoff_test.go index 735409ee902..c465148f431 100644 --- a/chasm/lib/scheduler/migration_handoff_test.go +++ b/chasm/lib/scheduler/migration_handoff_test.go @@ -36,8 +36,8 @@ type migrationCloseFaultEngine struct { func requireMigrationCounterexamples(t *testing.T) { t.Helper() - if os.Getenv("TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES") == "" { - t.Skip("set TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES=1 to run known failing migration repros") + if os.Getenv("TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES") != "1" { + t.Skip("known migration counterexample; set TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES=1 to run") } } @@ -108,15 +108,19 @@ func TestMigrationScenario_RollbackRetryBoundaries(t *testing.T) { faultEngine := &migrationCloseFaultEngine{Engine: e.engine, armed: boundary == "before_source_close" || boundary == "after_source_close", afterCommit: boundary == "after_source_close"} var destination *schedulespb.StartScheduleArgs + var firstRequestID string attempts, creations := 0, 0 historyClient.EXPECT().StartWorkflowExecution(gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn( func(_ context.Context, req *historyservice.StartWorkflowExecutionRequest, _ ...grpc.CallOption) (*historyservice.StartWorkflowExecutionResponse, error) { attempts++ + if attempts == 1 { + firstRequestID = req.StartRequest.RequestId + } if attempts == 1 && boundary == "before_destination" { return nil, serviceerror.NewUnavailable("injected before destination commit") } if destination != nil { - return nil, serviceerror.NewWorkflowExecutionAlreadyStarted("retry", "first-request", "destination-run") + return nil, serviceerror.NewWorkflowExecutionAlreadyStarted("retry", firstRequestID, "destination-run") } destination = &schedulespb.StartScheduleArgs{} require.NoError(t, sdk.PreferProtoDataConverter.FromPayloads(req.StartRequest.Input, destination)) diff --git a/chasm/lib/scheduler/migration_rollback_phase_test.go b/chasm/lib/scheduler/migration_rollback_phase_test.go index 6adc46e86a8..8e1c811438c 100644 --- a/chasm/lib/scheduler/migration_rollback_phase_test.go +++ b/chasm/lib/scheduler/migration_rollback_phase_test.go @@ -1,7 +1,6 @@ package scheduler_test import ( - "os" "testing" "time" @@ -36,9 +35,7 @@ func TestRollbackPhaseNativeControl(t *testing.T) { } func TestRollbackPhaseCounterexample(t *testing.T) { - if os.Getenv("TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES") != "1" { - t.Skip("set TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES=1") - } + requireMigrationCounterexamples(t) testRollbackPhaseGate(t) } @@ -51,7 +48,7 @@ func testRollbackPhaseGate(t *testing.T) { i.InvokerState = rollbackPhaseState(phase) before := common.CloneProto(i.InvokerState) _, err := s.MigrateToWorkflow(ctx, &schedulerpb.MigrateToWorkflowRequest{}) - require.Error(t, err, "seed=rollback-%s: V1 cannot execute persisted invoker phase; retain native task ownership", phase) + require.Error(t, err, "case=%s: V1 cannot execute persisted invoker phase; retain native task ownership", phase) require.Nil(t, s.WorkflowMigration) require.False(t, s.Schedule.State.Paused) protorequire.ProtoEqual(t, before, i.InvokerState) @@ -82,8 +79,8 @@ func TestRollbackPhaseConversionPreservesFieldsButLosesQueues(t *testing.T) { s, ctx, _ := setupSchedulerForTest(t) i := rollbackPhaseState("mixed") args := migration.CHASMToLegacyStartScheduleArgs(s.SchedulerState, s.Generator.Get(ctx).GeneratorState, i, nil, nil, nil, nil, ctx.Now(s)) + // InternalState has no cancel/terminate queues, so only buffered starts carry over. + require.Len(t, args.State.BufferedStarts, 1) protorequire.ProtoEqual(t, i.BufferedStarts[0], args.State.BufferedStarts[0]) require.Empty(t, args.Info.RunningWorkflows) - require.Len(t, i.CancelWorkflows, 1) - require.Len(t, i.TerminateWorkflows, 1) } diff --git a/chasm/lib/scheduler/schedulertest/driver.go b/chasm/lib/scheduler/schedulertest/driver.go index 937b90d3c00..da405adb7d9 100644 --- a/chasm/lib/scheduler/schedulertest/driver.go +++ b/chasm/lib/scheduler/schedulertest/driver.go @@ -244,7 +244,7 @@ func newDriverScaffold(t *testing.T, opts ...DriverOption) (*Driver, *driverConf startRequests[req.GetRequestId()] = struct{}{} rpcMu.Unlock() if startCalls != nil { - call := &StartCall{Request: common.CloneProto(req), result: make(chan startResponse, 1), ledger: external} + call := &StartCall{Request: common.CloneProto(req), t: t, result: make(chan startResponse, 1), ledger: external} select { case startCalls <- call: case <-ctx.Done(): diff --git a/chasm/lib/scheduler/schedulertest/external.go b/chasm/lib/scheduler/schedulertest/external.go index 8993e35dc7f..1e3054e0fdd 100644 --- a/chasm/lib/scheduler/schedulertest/external.go +++ b/chasm/lib/scheduler/schedulertest/external.go @@ -5,6 +5,7 @@ import ( "fmt" "maps" "sync" + "testing" commonpb "go.temporal.io/api/common/v1" "go.temporal.io/api/workflowservice/v1" @@ -49,6 +50,7 @@ func (l *externalLedger) snapshot() map[string]ExternalStart { // test can deliver a callback or lose the response after external acceptance. type StartCall struct { Request *workflowservice.StartWorkflowExecutionRequest + t testing.TB result chan startResponse ledger *externalLedger } @@ -61,7 +63,11 @@ type startResponse struct { func (c *StartCall) Accept() (ExternalStart, error) { return c.ledger.accept(c.Request) } func (c *StartCall) Respond(start ExternalStart, err error) { - c.result <- startResponse{response: &workflowservice.StartWorkflowExecutionResponse{RunId: start.RunID}, err: err} + select { + case c.result <- startResponse{response: &workflowservice.StartWorkflowExecutionResponse{RunId: start.RunID}, err: err}: + default: + c.t.Fatal("StartCall.Respond called more than once") + } } // WithControlledStarts makes start RPCs wait for explicit Accept/Respond events. @@ -124,6 +130,8 @@ func (d *Driver) FinishSideEffect() { if err := call.Record(); err != nil { d.T.Fatalf("record start result: %v", err) } + case call := <-d.startCalls: + d.T.Fatalf("unconsumed start RPC for request %s; call NextStart before FinishSideEffect", call.Request.GetRequestId()) case err := <-d.sideEffectDone: d.sideEffectDone = nil d.finishSideEffect(d.actionsBefore, err) diff --git a/chasm/lib/scheduler/schedulertest/invariants_test.go b/chasm/lib/scheduler/schedulertest/invariants_test.go index 8997218eb7c..7011925de5e 100644 --- a/chasm/lib/scheduler/schedulertest/invariants_test.go +++ b/chasm/lib/scheduler/schedulertest/invariants_test.go @@ -80,7 +80,7 @@ func TestInvariants_HoldAcrossScenarios(t *testing.T) { // flagged as stuck. func TestCheckInvariants_DetectsStuck(t *testing.T) { stuck := schedulertest.Snapshot{ - Now: time.Now(), + Now: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC), HasPendingTask: false, Closed: false, IsHeldOpen: false, @@ -94,7 +94,7 @@ func TestCheckInvariants_DetectsStuck(t *testing.T) { // no pending task is NOT flagged (legitimate hold-open). func TestCheckInvariants_AllowsHeldOpenWithoutTask(t *testing.T) { heldOpen := schedulertest.Snapshot{ - Now: time.Now(), + Now: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC), HasPendingTask: false, Closed: false, Paused: true, diff --git a/service/frontend/migration_delete_race_test.go b/service/frontend/migration_delete_race_test.go index 0c0f617df0c..86751e03ce1 100644 --- a/service/frontend/migration_delete_race_test.go +++ b/service/frontend/migration_delete_race_test.go @@ -2,7 +2,9 @@ package frontend import ( "context" + "errors" "os" + "sync" "testing" "time" @@ -42,7 +44,7 @@ func (c *migrationDeleteRaceClient) DeleteSchedule(_ context.Context, req *sched func TestMigrationDeleteNativeControl(t *testing.T) { testMigrationDeleteRace(t, false) } func TestMigrationDeleteCounterexample(t *testing.T) { if os.Getenv("TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES") != "1" { - t.Skip("set TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES=1") + t.Skip("known migration counterexample; set TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES=1 to run") } testMigrationDeleteRace(t, true) } @@ -71,7 +73,11 @@ func testMigrationDeleteRace(t *testing.T, inFlight bool) { return err } started, release, created := make(chan struct{}), make(chan struct{}), make(chan error, 1) + var releaseOnce sync.Once + releaseCreate := func() { releaseOnce.Do(func() { close(release) }) } if inFlight { + // Unblock the create goroutine even if the test fails early. + t.Cleanup(releaseCreate) go func() { close(started); <-release; created <- create() }() <-started } else { @@ -88,16 +94,16 @@ func testMigrationDeleteRace(t *testing.T, inFlight bool) { handler := &WorkflowHandler{logger: logger, namespaceRegistry: namespaces, historyClient: history, schedulerClient: &migrationDeleteRaceClient{engineCtx: engineCtx, ref: ref}, config: &Config{EnableSchedules: func(string) bool { return true }, EnableCHASMSchedulerCreation: func(string) bool { return true }}} _, deleteErr := handler.DeleteSchedule(context.Background(), &workflowservice.DeleteScheduleRequest{Namespace: "ns", ScheduleId: key.BusinessID}) if inFlight { - close(release) + releaseCreate() require.NoError(t, <-created) } require.NoError(t, deleteErr) _, err := chasm.ReadComponent(engineCtx, ref, func(s *chasmscheduler.Scheduler, _ chasm.Context, _ struct{}) (struct{}, error) { - require.True(t, s.Closed, "seed=delete-create: successful DeleteSchedule must not be followed by an active migrated scheduler") + require.True(t, s.Closed, "successful DeleteSchedule must not be followed by an active migrated scheduler") return struct{}{}, nil }, struct{}{}) if err != nil { - var notFound *serviceerror.NotFound - require.ErrorAs(t, err, ¬Found) + _, ok := errors.AsType[*serviceerror.NotFound](err) + require.True(t, ok, "unexpected error: %v", err) } } diff --git a/service/worker/scheduler/migration_scenarios_test.go b/service/worker/scheduler/migration_scenarios_test.go index 9f6c9f2561d..a16b4ab5210 100644 --- a/service/worker/scheduler/migration_scenarios_test.go +++ b/service/worker/scheduler/migration_scenarios_test.go @@ -51,8 +51,8 @@ func newMigrationScenario(t *testing.T) (*testsuite.TestWorkflowEnvironment, *sc func requireMigrationCounterexamples(t *testing.T) { t.Helper() - if os.Getenv("TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES") == "" { - t.Skip("set TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES=1 to run known failing migration repros") + if os.Getenv("TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES") != "1" { + t.Skip("known migration counterexample; set TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES=1 to run") } } @@ -105,7 +105,7 @@ func checkMigrationSignalAccounting(t *testing.T, duringMigration bool) { require.True(t, delivered) require.NotNil(t, destination) pending := 0 - for _, start := range destination.InvokerState.BufferedStarts { + for _, start := range destination.GetInvokerState().GetBufferedStarts() { if start.Manual && start.RunId == "" && start.Completed == nil { pending++ } @@ -260,11 +260,9 @@ func checkMigrationCommitAccounting(t *testing.T, commit, disable bool, version require.NotEmpty(t, sourceRequests) require.NotNil(t, client.destination) manualAtDestination := 0 - for _, start := range client.destination.InvokerState.BufferedStarts { + for _, start := range client.destination.GetInvokerState().GetBufferedStarts() { if start.Manual && start.RunId == "" && start.Completed == nil { manualAtDestination++ - t.Logf("destination workflow/request=%s/%s; source workflow/request=%s/%s", start.WorkflowId, start.RequestId, - sourceRequests[0].Request.WorkflowId, sourceRequests[0].Request.RequestId) } } require.Zero(t, manualAtDestination, "source executed the manual action after migration returned an error; the committed destination must not still own it") From ca0ded1ee9671240349f00808cdd3b4b7b651368 Mon Sep 17 00:00:00 2001 From: chaptersix <13949480+chaptersix@users.noreply.github.com> Date: Wed, 11 Nov 2026 16:24:15 -0600 Subject: [PATCH 04/22] Run property tests in a dedicated CI job The unit-test job retries failures, and each retry draws a new Rapid seed, so a rare property violation can be retried away. Add a property-test target that runs more generated cases with no retries, and a CI job that uploads the minimized failing cases. Co-Authored-By: Claude Fable 5.2 --- .github/workflows/run-tests.yml | 28 ++++++++++++++++++++++++++++ Makefile | 14 ++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 0a9fdd3d7c2..1015d1dbbfd 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -324,6 +324,33 @@ jobs: path: ${{ env.LEAK_OUTPUT_DIR }} if-no-files-found: ignore + property-test: + name: Property test + needs: [pre-build] + runs-on: ubuntu-24.04-arm + env: + GITHUB_TIMEOUT: 15 + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + ref: ${{ env.COMMIT }} + + - name: Prepare Go + uses: ./.github/actions/prepare-go + + - name: Run property tests + timeout-minutes: ${{ fromJSON(env.GITHUB_TIMEOUT) }} + run: make PROPERTY_TEST_TIMEOUT="$(("$GITHUB_TIMEOUT" - 1))m" property-test + + - name: Upload failing cases + if: failure() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: property-test-failfiles + path: "**/testdata/rapid/**/*.fail" + if-no-files-found: ignore + integration-test: name: Integration test needs: [pre-build] @@ -511,6 +538,7 @@ jobs: needs: - misc-checks - unit-test + - property-test - integration-test - functional-test runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index 6c881c70323..28b54fc2e12 100644 --- a/Makefile +++ b/Makefile @@ -516,6 +516,20 @@ leak-test: -timeout $(LEAK_TIMEOUT) $(TEST_TAG_FLAG) \ ./tests/leakcheck/ -args -persistenceType=sql -persistenceDriver=sqlite +# Property tests get a dedicated run: more generated cases than the unit-test +# default, and no retries, so a failing seed is reported instead of retried away. +# Failing cases are minimized into testdata/rapid/ next to the test. +PROPERTY_TEST_DIRS ?= ./chasm/lib/scheduler/schedulertest/... +PROPERTY_TEST_CHECKS ?= 5000 +PROPERTY_TEST_TIMEOUT ?= 10m +property-test: + @printf $(COLOR) "Run property tests..." + CGO_ENABLED=$(TEST_CGO_ENABLED) go test $(PROPERTY_TEST_DIRS) -count=1 \ + -timeout $(PROPERTY_TEST_TIMEOUT) \ + $(if $(TEST_RACE_ENABLED),-race,) \ + $(TEST_TAG_FLAG) \ + -rapid.checks=$(PROPERTY_TEST_CHECKS) $(TEST_ARGS) + verify-test-log: @test -s test.log || (echo "TEST FAILURE: test.log is missing or empty" && exit 1) @grep -q "^ok" test.log || (echo "TEST FAILURE: no passing test found in test.log" && exit 1) From 608344e29c6217ff434cb39ab56b9a046ba43f22 Mon Sep 17 00:00:00 2001 From: chaptersix <13949480+chaptersix@users.noreply.github.com> Date: Wed, 11 Nov 2026 16:32:48 -0600 Subject: [PATCH 05/22] Fix scheduler backfill migration cursor and callback ordering V1 to CHASM migration dropped the V1 backfill cursor, so the backfiller re-included the already-consumed start point. Migrated backfillers now carry the V1 StartTime as LastProcessedTime, and processBackfill treats a non-nil LastProcessedTime (including the Unix epoch) as recorded progress instead of treating a zero timestamp as unset. Migrated backfillers also got an immediate task before SchedulerCallbacksTask reconciled imported running workflows, so they could buffer starts against stale state. When callbacks are needed, the backfillers now start without a task and SchedulerCallbacksTask schedules them after reconciliation. CHASM to V1 rollback only overrode StartTime when Attempt > 0, so a fresh [T,T] backfill lost its inclusive boundary on V1's exclusive cursor. Rollback now uses the recorded cursor if present, else shifts the range start back 1ms. Co-Authored-By: Claude Fable 5.2 --- chasm/lib/scheduler/backfiller.go | 23 +++++++++++++++---- chasm/lib/scheduler/backfiller_tasks.go | 15 ++++-------- chasm/lib/scheduler/migration/migration.go | 15 +++++++++--- .../lib/scheduler/migration_backfill_test.go | 7 +++--- chasm/lib/scheduler/scheduler.go | 13 +++++++++-- chasm/lib/scheduler/scheduler_tasks.go | 3 +++ .../scheduler/migration_scenarios_test.go | 1 - 7 files changed, 53 insertions(+), 24 deletions(-) diff --git a/chasm/lib/scheduler/backfiller.go b/chasm/lib/scheduler/backfiller.go index ba26dcb4b82..72b28b7e977 100644 --- a/chasm/lib/scheduler/backfiller.go +++ b/chasm/lib/scheduler/backfiller.go @@ -37,7 +37,7 @@ func addBackfiller( ) *Backfiller { id := schedulerinternal.GenerateBackfillerID() // LastProcessedTime is intentionally left unset here. For range backfills it - // is the "progress recorded" signal and must stay zero until a batch is + // is the "progress recorded" signal and must stay nil until a batch is // actually processed (see processBackfill). Trigger backfills, which use it as // their fire time, set it explicitly in NewImmediateBackfiller. backfiller := newBackfillerWithState(ctx, &schedulerpb.BackfillerState{ @@ -55,12 +55,27 @@ func addBackfiller( } func newBackfillerWithState(ctx chasm.MutableContext, state *schedulerpb.BackfillerState) *Backfiller { - backfiller := &Backfiller{ + backfiller := newBackfillerWithStateWithoutTask(ctx, state) + backfiller.scheduleInitialTask(ctx) + return backfiller +} + +// newBackfillerWithStateWithoutTask returns a Backfiller that has no task +// scheduled. Call scheduleInitialTask to start it. +func newBackfillerWithStateWithoutTask(ctx chasm.MutableContext, state *schedulerpb.BackfillerState) *Backfiller { + return &Backfiller{ BackfillerState: state, EventLog: chasm.NewComponentField(ctx, NewEventLog(ctx)), } - backfiller.scheduleTask(ctx, chasm.TaskScheduledTimeImmediate) - return backfiller +} + +// scheduleInitialTask schedules the first immediate task, unless a task was +// already scheduled. +func (b *Backfiller) scheduleInitialTask(ctx chasm.MutableContext) { + if b.GetTaskStamp() != 0 { + return + } + b.scheduleTask(ctx, chasm.TaskScheduledTimeImmediate) } // scheduleTask advances the stamp for the initial task or a successor. diff --git a/chasm/lib/scheduler/backfiller_tasks.go b/chasm/lib/scheduler/backfiller_tasks.go index 37ee41d3134..c68d5392208 100644 --- a/chasm/lib/scheduler/backfiller_tasks.go +++ b/chasm/lib/scheduler/backfiller_tasks.go @@ -176,11 +176,13 @@ func (b *BackfillerTaskHandler) processBackfill( endTime := request.GetEndTime().AsTime() // Resume from the high watermark only once genuine progress has been recorded. - // The watermark is left unset until a batch is actually processed (see Execute), - // so a fresh or capacity-stalled backfiller starts from the range start. + // The watermark stays nil until a batch is actually processed (see Execute) or + // a V1 cursor is imported, so a fresh or capacity-stalled backfiller starts + // from the range start. Presence, not value, matters: an explicit Unix epoch + // cursor is real progress. var startTime time.Time lastProcessed := backfiller.GetLastProcessedTime() - if hasRecordedProgress(lastProcessed) { + if lastProcessed != nil { startTime = lastProcessed.AsTime() } else { // On the first attempt, start slightly behind to make the range inclusive. @@ -213,13 +215,6 @@ func (b *BackfillerTaskHandler) processBackfill( return } -// hasRecordedProgress reports whether a backfiller's high watermark reflects a -// batch that was actually processed. An unset (nil or zero) watermark means no -// progress yet - a fresh backfiller. -func hasRecordedProgress(lastProcessed *timestamppb.Timestamp) bool { - return lastProcessed != nil && (lastProcessed.GetSeconds() != 0 || lastProcessed.GetNanos() != 0) -} - // backoffDelay returns the amount of delay that should be added when retrying. func (b *BackfillerTaskHandler) backoffDelay(backfiller *Backfiller) time.Duration { // Increment GetAttempt here early, to avoid needing to increment diff --git a/chasm/lib/scheduler/migration/migration.go b/chasm/lib/scheduler/migration/migration.go index 88c63b8b453..e294978749f 100644 --- a/chasm/lib/scheduler/migration/migration.go +++ b/chasm/lib/scheduler/migration/migration.go @@ -436,12 +436,16 @@ func convertBackfillsLegacyToCHASM( for _, v1Backfill := range legacyBackfills { backfillID := schedulerinternal.GenerateBackfillerID() + // V1's StartTime is an exclusive cursor that has already been consumed, so + // carry it over as the CHASM high watermark. The watermark's presence marks + // the backfiller as having recorded progress, which stops processBackfill + // from re-including the cursor point. backfillers[backfillID] = &schedulerpb.BackfillerState{ Request: &schedulerpb.BackfillerState_BackfillRequest{ BackfillRequest: common.CloneProto(v1Backfill), }, BackfillId: backfillID, - LastProcessedTime: nil, + LastProcessedTime: common.CloneProto(v1Backfill.GetStartTime()), Attempt: 0, } } @@ -551,8 +555,13 @@ func convertBackfillersCHASMToLegacy( for _, backfiller := range backfillers { if request := backfiller.GetBackfillRequest(); request != nil { backfill := common.CloneProto(request) - if backfiller.GetAttempt() > 0 && backfiller.GetLastProcessedTime() != nil { - backfill.StartTime = common.CloneProto(backfiller.GetLastProcessedTime()) + // V1's StartTime is an exclusive cursor. Recorded progress is already + // such a cursor. A fresh backfiller's range start is inclusive, so shift + // it back to keep that point. + if lastProcessed := backfiller.GetLastProcessedTime(); lastProcessed != nil { + backfill.StartTime = common.CloneProto(lastProcessed) + } else { + backfill.StartTime = timestamppb.New(request.GetStartTime().AsTime().Add(-time.Millisecond)) } ongoing = append(ongoing, backfill) continue diff --git a/chasm/lib/scheduler/migration_backfill_test.go b/chasm/lib/scheduler/migration_backfill_test.go index fb175bea16c..1b69f93c9f4 100644 --- a/chasm/lib/scheduler/migration_backfill_test.go +++ b/chasm/lib/scheduler/migration_backfill_test.go @@ -29,7 +29,6 @@ func TestMigrationBackfill_NativeResumesAfterWatermark(t *testing.T) { } func TestMigrationBackfill_MigratedResumesAfterWatermark(t *testing.T) { - requireMigrationCounterexamples(t) for _, end := range []time.Time{time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC), time.Unix(60, 0).UTC()} { t.Run(end.String(), func(t *testing.T) { checkMigrationBackfillWatermark(t, true, end) }) } @@ -83,7 +82,6 @@ func checkMigrationBackfillWatermark(t *testing.T, migrate bool, when time.Time) } func TestMigrationBackfill_WaitsForCallbackReconciliation(t *testing.T) { - requireMigrationCounterexamples(t) boundary := time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC) logger := testlogger.NewTestLogger(t, testlogger.FailOnExpectedErrorOnly) timeSource := clock.NewEventTimeSource().Update(boundary.Add(time.Hour)) @@ -152,8 +150,9 @@ func TestMigrationBackfill_WaitsForCallbackReconciliation(t *testing.T) { nil, ) callbackHandler := scheduler.NewSchedulerCallbacksTaskHandler(scheduler.SchedulerCallbacksTaskHandlerOptions{ - Config: defaultConfig(), - HistoryClient: historyClient, + Config: defaultConfig(), + HistoryClient: historyClient, + MetricsHandler: metrics.NoopMetricsHandler, }) require.NoError(t, callbackHandler.Execute( engineCtx, diff --git a/chasm/lib/scheduler/scheduler.go b/chasm/lib/scheduler/scheduler.go index e97083d2151..44af5644b11 100644 --- a/chasm/lib/scheduler/scheduler.go +++ b/chasm/lib/scheduler/scheduler.go @@ -325,8 +325,17 @@ func CreateSchedulerFromMigration( sched.Invoker = chasm.NewComponentField(ctx, newInvokerWithState(ctx, state.GetInvokerState())) sched.Generator = chasm.NewComponentField(ctx, newGeneratorWithState(ctx, state.GetGeneratorState())) + // Backfillers also wait for SchedulerCallbacksTask when it is needed, so they + // don't buffer starts against stale running-workflow state. + needsCallbacks := slices.ContainsFunc(state.GetInvokerState().GetBufferedStarts(), needsCallback) for backfillID, backfillerState := range state.GetBackfillers() { - sched.Backfillers[backfillID] = chasm.NewComponentField(ctx, newBackfillerWithState(ctx, backfillerState)) + var backfiller *Backfiller + if needsCallbacks { + backfiller = newBackfillerWithStateWithoutTask(ctx, backfillerState) + } else { + backfiller = newBackfillerWithState(ctx, backfillerState) + } + sched.Backfillers[backfillID] = chasm.NewComponentField(ctx, backfiller) } visibility, err := chasm.NewVisibilityWithData(ctx, state.GetSearchAttributes(), state.GetMemo()) @@ -337,7 +346,7 @@ func CreateSchedulerFromMigration( // Defer generation until SchedulerCallbacksTask resolves stale running-workflow // state; if there are none, generate directly. - if slices.ContainsFunc(state.GetInvokerState().GetBufferedStarts(), needsCallback) { + if needsCallbacks { ctx.AddTask(sched, chasm.TaskAttributes{}, &schedulerpb.SchedulerCallbacksTask{}) } else { sched.Generator.Get(ctx).Generate(ctx) diff --git a/chasm/lib/scheduler/scheduler_tasks.go b/chasm/lib/scheduler/scheduler_tasks.go index 4d0d5506f34..604839fa74d 100644 --- a/chasm/lib/scheduler/scheduler_tasks.go +++ b/chasm/lib/scheduler/scheduler_tasks.go @@ -237,6 +237,9 @@ func (r *SchedulerCallbacksTaskHandler) Execute( // Now that running workflow state has been refreshed, scheduler tasks can be // fired. + for _, backfillerField := range s.Backfillers { + backfillerField.Get(ctx).scheduleInitialTask(ctx) + } invoker.addTasks(ctx) generator.Generate(ctx) diff --git a/service/worker/scheduler/migration_scenarios_test.go b/service/worker/scheduler/migration_scenarios_test.go index a16b4ab5210..04549ab5116 100644 --- a/service/worker/scheduler/migration_scenarios_test.go +++ b/service/worker/scheduler/migration_scenarios_test.go @@ -151,7 +151,6 @@ func TestMigrationScenario_NativeBackfillIncludesStart(t *testing.T) { } func TestMigrationCounterexample_RollbackFreshBackfill(t *testing.T) { - requireMigrationCounterexamples(t) checkRollbackBackfillStart(t, true) } From 3fb398378d57e207f5e2dc663d25e50988e0af49 Mon Sep 17 00:00:00 2001 From: chaptersix <13949480+chaptersix@users.noreply.github.com> Date: Wed, 11 Nov 2026 16:35:06 -0600 Subject: [PATCH 06/22] Refuse rollback while invoker actions are in flight Rolling a CHASM schedule back to V1 silently dropped buffered starts that were ready or retrying (Attempt > 0, no run recorded) and pending cancel/terminate requests, because V1 InternalState has no place for them. Add Invoker.canMigrateToWorkflow and check it in MigrateToWorkflow and in the migrate task. Both return the retryable ErrMigrationActionsPending (Unavailable, like ErrMigrationPending) until the invoker drains, after which migration succeeds. No persisted state or proto changes. The schedulertest ledger test now asserts refusal then success after draining; the migration-untransferable invariant is kept as a defence. Co-Authored-By: Claude Fable 5.2 --- chasm/lib/scheduler/invoker.go | 14 +++++ .../migration_rollback_phase_test.go | 58 ++++++++++++++++--- chasm/lib/scheduler/scheduler.go | 8 ++- chasm/lib/scheduler/scheduler_migrate_task.go | 4 ++ .../scheduler/schedulertest/inflight_test.go | 26 ++++++++- 5 files changed, 99 insertions(+), 11 deletions(-) diff --git a/chasm/lib/scheduler/invoker.go b/chasm/lib/scheduler/invoker.go index 0cc6a35a773..e649388c6a6 100644 --- a/chasm/lib/scheduler/invoker.go +++ b/chasm/lib/scheduler/invoker.go @@ -28,6 +28,20 @@ type Invoker struct { EventLog chasm.Field[*EventLog] } +// canMigrateToWorkflow reports whether V1 can take over this invoker's work. +// V1 state has no place for pending cancel/terminate requests or for starts that +// are ready or retrying, so those would be lost on migration. A start with +// Attempt > 0 becomes transferable once it records a RunId or a Completed +// result, or once processing resets its Attempt to 0 or -1. +func (i *Invoker) canMigrateToWorkflow() bool { + if len(i.CancelWorkflows) > 0 || len(i.TerminateWorkflows) > 0 { + return false + } + return !slices.ContainsFunc(i.BufferedStarts, func(start *schedulespb.BufferedStart) bool { + return start.GetAttempt() > 0 && start.GetRunId() == "" && start.GetCompleted() == nil + }) +} + func (i *Invoker) LifecycleState(ctx chasm.Context) chasm.LifecycleState { return chasm.LifecycleStateRunning } diff --git a/chasm/lib/scheduler/migration_rollback_phase_test.go b/chasm/lib/scheduler/migration_rollback_phase_test.go index 8e1c811438c..ba15644c8ab 100644 --- a/chasm/lib/scheduler/migration_rollback_phase_test.go +++ b/chasm/lib/scheduler/migration_rollback_phase_test.go @@ -13,6 +13,7 @@ import ( schedulerpb "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" "go.temporal.io/server/chasm/lib/scheduler/migration" "go.temporal.io/server/common" + "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/testing/protorequire" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -34,13 +35,7 @@ func TestRollbackPhaseNativeControl(t *testing.T) { } } -func TestRollbackPhaseCounterexample(t *testing.T) { - requireMigrationCounterexamples(t) - testRollbackPhaseGate(t) -} - -func testRollbackPhaseGate(t *testing.T) { - t.Helper() +func TestRollbackPhaseGate(t *testing.T) { for _, phase := range []string{"ready", "retry", "cancel", "terminate", "mixed"} { t.Run(phase, func(t *testing.T) { s, ctx, _ := setupSchedulerForTest(t) @@ -48,7 +43,7 @@ func testRollbackPhaseGate(t *testing.T) { i.InvokerState = rollbackPhaseState(phase) before := common.CloneProto(i.InvokerState) _, err := s.MigrateToWorkflow(ctx, &schedulerpb.MigrateToWorkflowRequest{}) - require.Error(t, err, "case=%s: V1 cannot execute persisted invoker phase; retain native task ownership", phase) + require.ErrorIs(t, err, scheduler.ErrMigrationActionsPending, "case=%s: V1 cannot execute persisted invoker phase; retain native task ownership", phase) require.Nil(t, s.WorkflowMigration) require.False(t, s.Schedule.State.Paused) protorequire.ProtoEqual(t, before, i.InvokerState) @@ -84,3 +79,50 @@ func TestRollbackPhaseConversionPreservesFieldsButLosesQueues(t *testing.T) { protorequire.ProtoEqual(t, i.BufferedStarts[0], args.State.BufferedStarts[0]) require.Empty(t, args.Info.RunningWorkflows) } + +func TestRollbackPhaseTransferable(t *testing.T) { + for _, phase := range []string{"unprocessed", "deferred", "running", "completed", "drained"} { + t.Run(phase, func(t *testing.T) { + s, ctx, _ := setupSchedulerForTest(t) + i := s.Invoker.Get(ctx) + i.InvokerState = rollbackPhaseState("ready") + start := i.BufferedStarts[0] + switch phase { + case "unprocessed": + start.Attempt = 0 + case "deferred": + start.Attempt = -1 + case "running": + start.RunId = "run" + case "completed": + start.RunId = "run" + start.Completed = &schedulespb.CompletedResult{} + case "drained": + i.BufferedStarts = nil + default: + t.Fatalf("unknown phase %q", phase) + } + _, err := s.MigrateToWorkflow(ctx, &schedulerpb.MigrateToWorkflowRequest{}) + require.NoError(t, err) + require.NotNil(t, s.WorkflowMigration) + }) + } +} + +func TestRollbackPhaseMigrateTaskRefusesPendingActions(t *testing.T) { + e := newSchedulerTestEngine(t, defaultSchedule()) + require.NoError(t, e.updateScheduler(func(s *scheduler.Scheduler, ctx chasm.MutableContext) error { + s.Invoker.Get(ctx).InvokerState = rollbackPhaseState("mixed") + s.WorkflowMigration = &schedulerpb.WorkflowMigrationState{} + s.Schedule.State.Paused = true + return nil + })) + h := scheduler.NewSchedulerMigrateToWorkflowTaskHandler(scheduler.SchedulerMigrateToWorkflowTaskHandlerOptions{Config: defaultConfig(), MetricsHandler: metrics.NoopMetricsHandler, BaseLogger: e.logger}) + err := h.Execute(e.engineCtx, e.rootRef, chasm.TaskAttributes{}, &schedulerpb.SchedulerMigrateToWorkflowTask{}) + require.ErrorIs(t, err, scheduler.ErrMigrationActionsPending) + require.NoError(t, e.readScheduler(func(s *scheduler.Scheduler, _ chasm.Context) error { + require.False(t, s.Closed) + require.NotNil(t, s.WorkflowMigration) + return nil + })) +} diff --git a/chasm/lib/scheduler/scheduler.go b/chasm/lib/scheduler/scheduler.go index 44af5644b11..e2b3956597b 100644 --- a/chasm/lib/scheduler/scheduler.go +++ b/chasm/lib/scheduler/scheduler.go @@ -115,7 +115,9 @@ var ( ErrSentinel = serviceerror.NewNotFound("schedule is a sentinel") ErrSentinelBlocked = serviceerror.NewUnavailable("schedule is a sentinel; please retry after sentinel expires") ErrMigrationPending = serviceerror.NewUnavailable("schedule has a pending migration to workflow; please retry later") - ErrTimeSkippingMigration = serviceerror.NewFailedPrecondition("schedule with time skipping cannot migrate to workflow-backed scheduler") + // ErrMigrationActionsPending is retryable: the invoker finishes the actions and a later attempt succeeds. + ErrMigrationActionsPending = serviceerror.NewUnavailable("schedule has in-flight actions; retry migration after they finish") + ErrTimeSkippingMigration = serviceerror.NewFailedPrecondition("schedule with time skipping cannot migrate to workflow-backed scheduler") ) // NewScheduler returns an initialized CHASM scheduler root component. @@ -888,6 +890,10 @@ func (s *Scheduler) MigrateToWorkflow( return nil, ErrTimeSkippingMigration } + if !s.Invoker.Get(ctx).canMigrateToWorkflow() { + return nil, ErrMigrationActionsPending + } + // Save pre-migration paused state, mark migration as pending, then pause. s.WorkflowMigration = &schedulerpb.WorkflowMigrationState{ PreMigrationPaused: s.Schedule.State.Paused, diff --git a/chasm/lib/scheduler/scheduler_migrate_task.go b/chasm/lib/scheduler/scheduler_migrate_task.go index 536b41b4fc8..60376647ff4 100644 --- a/chasm/lib/scheduler/scheduler_migrate_task.go +++ b/chasm/lib/scheduler/scheduler_migrate_task.go @@ -119,6 +119,10 @@ func (h *SchedulerMigrateToWorkflowTaskHandler) Execute( ctx, schedulerRef, func(s *Scheduler, ctx chasm.Context, _ any) (struct{}, error) { + // The task retries until the invoker has drained. + if !s.Invoker.Get(ctx).canMigrateToWorkflow() { + return struct{}{}, ErrMigrationActionsPending + } now := ctx.Now(s) schedulerState := common.CloneProto(s.SchedulerState) generatorState := common.CloneProto(s.Generator.Get(ctx).GeneratorState) diff --git a/chasm/lib/scheduler/schedulertest/inflight_test.go b/chasm/lib/scheduler/schedulertest/inflight_test.go index fcd8208fcb0..ae1c8cc1aec 100644 --- a/chasm/lib/scheduler/schedulertest/inflight_test.go +++ b/chasm/lib/scheduler/schedulertest/inflight_test.go @@ -68,13 +68,35 @@ func TestSelectedStartReservationCounterexample(t *testing.T) { require.Equal(t, 2, selected, "known counterexample: two BUFFER_ALL starts selected before the first acknowledgement; replace with <=1 when fixed") } -func TestMigrationCannotExemptExecutingObligations(t *testing.T) { +// Migration is refused while a start is ready to execute, since V1 cannot resume +// it, and succeeds once the invoker has started it. The migration-untransferable +// invariant stays as a defence in case the refusal regresses. +func TestMigrationRefusedWhileExecutingObligations(t *testing.T) { t.Parallel() schedule := DefaultSchedule() schedule.State.Paused = true d := NewDriver(t, WithSchedule(schedule)) require.NoError(t, d.TriggerImmediately()) + require.ErrorIs(t, d.MigrateToWorkflow(), scheduler.ErrMigrationActionsPending) + snap := d.Snapshot() + require.False(t, snap.Migrating) + require.Equal(t, 1, snap.UntransferableActions) + require.Empty(t, CheckInvariants(nil, snap)) + + for range 20 { + if d.Snapshot().UntransferableActions == 0 { + break + } + require.True(t, d.Step()) + } + require.Equal(t, 0, d.Snapshot().UntransferableActions) require.NoError(t, d.MigrateToWorkflow()) - violations := CheckInvariants(nil, d.Snapshot()) + require.True(t, d.Snapshot().Migrating) + require.Empty(t, CheckInvariants(nil, d.Snapshot())) +} + +func TestMigrationUntransferableInvariant(t *testing.T) { + t.Parallel() + violations := CheckInvariants(nil, Snapshot{Migrating: true, UntransferableActions: 1}) require.Contains(t, violations, Violation{Name: "migration-untransferable", Message: "1 execution/retry/cancel/terminate obligations lost native task ownership"}) } From 522ff18ec2d0018016143ed29b1699d3f9326e55 Mon Sep 17 00:00:00 2001 From: chaptersix <13949480+chaptersix@users.noreply.github.com> Date: Wed, 11 Nov 2026 16:39:02 -0600 Subject: [PATCH 07/22] Prove rollback destination ownership before closing the source The rollback migrate task sent a fresh random request ID on every attempt and treated any WorkflowExecutionAlreadyStarted as success. An unrelated workflow holding the V1 schedule's workflow ID was accepted as the destination and the CHASM source was closed. Persist a request_id on WorkflowMigrationState (new field 3) and send it on every start attempt. On AlreadyStarted, accept only when StartRequestId matches; otherwise walk to the chain's first run (the V1 workflow may have continued as new) and require its create request ID to match. A foreign chain or a missing first run returns an error and the source stays open. An Unavailable before the destination commits stays retryable. Compatibility: MigrateToWorkflow generates an ID when none is supplied, as does the admin handler. A migration persisted before this field has an empty ID; the task assigns and persists one before its first start attempt and never accepts AlreadyStarted on it without chain verification. Co-Authored-By: Claude Fable 5.2 --- .../gen/schedulerpb/v1/message.pb.go | 22 ++++- ...t.go => migration_chain_ownership_test.go} | 13 ++- chasm/lib/scheduler/migration_handoff_test.go | 88 +++++++++++++++++- chasm/lib/scheduler/proto/v1/message.proto | 6 ++ chasm/lib/scheduler/scheduler.go | 9 ++ chasm/lib/scheduler/scheduler_migrate_task.go | 91 ++++++++++++++++++- chasm/lib/scheduler/scheduler_migrate_test.go | 2 +- service/frontend/admin_handler.go | 7 +- service/frontend/admin_handler_test.go | 23 +++++ 9 files changed, 244 insertions(+), 17 deletions(-) rename chasm/lib/scheduler/{migration_chain_counterexample_test.go => migration_chain_ownership_test.go} (87%) diff --git a/chasm/lib/scheduler/gen/schedulerpb/v1/message.pb.go b/chasm/lib/scheduler/gen/schedulerpb/v1/message.pb.go index ed347794d8d..2e7c85789e4 100644 --- a/chasm/lib/scheduler/gen/schedulerpb/v1/message.pb.go +++ b/chasm/lib/scheduler/gen/schedulerpb/v1/message.pb.go @@ -188,8 +188,13 @@ type WorkflowMigrationState struct { PreMigrationPaused bool `protobuf:"varint,1,opt,name=pre_migration_paused,json=preMigrationPaused,proto3" json:"pre_migration_paused,omitempty"` // The schedule's notes before migration was initiated. PreMigrationNotes string `protobuf:"bytes,2,opt,name=pre_migration_notes,json=preMigrationNotes,proto3" json:"pre_migration_notes,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Request ID used for every StartWorkflowExecution attempt of the destination + // V1 workflow. It identifies the destination as created by this migration. + // Empty only on migrations persisted before this field existed; the migrate + // task assigns one before its first start attempt. + RequestId string `protobuf:"bytes,3,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *WorkflowMigrationState) Reset() { @@ -236,6 +241,13 @@ func (x *WorkflowMigrationState) GetPreMigrationNotes() string { return "" } +func (x *WorkflowMigrationState) GetRequestId() string { + if x != nil { + return x.RequestId + } + return "" +} + // CHASM scheduler's Generator internal state. type GeneratorState struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -774,10 +786,12 @@ const file_temporal_server_chasm_lib_scheduler_proto_v1_message_proto_rawDesc = " \x01(\bR\bsentinel\x12s\n" + "\x12workflow_migration\x18\v \x01(\v2D.temporal.server.chasm.lib.scheduler.proto.v1.WorkflowMigrationStateR\x11workflowMigration\x12B\n" + "\x0fidle_close_time\x18\f \x01(\v2\x1a.google.protobuf.TimestampR\ridleCloseTime\x12B\n" + - "\x0flast_event_time\x18\r \x01(\v2\x1a.google.protobuf.TimestampR\rlastEventTime\"z\n" + + "\x0flast_event_time\x18\r \x01(\v2\x1a.google.protobuf.TimestampR\rlastEventTime\"\x99\x01\n" + "\x16WorkflowMigrationState\x120\n" + "\x14pre_migration_paused\x18\x01 \x01(\bR\x12preMigrationPaused\x12.\n" + - "\x13pre_migration_notes\x18\x02 \x01(\tR\x11preMigrationNotes\"\xfd\x01\n" + + "\x13pre_migration_notes\x18\x02 \x01(\tR\x11preMigrationNotes\x12\x1d\n" + + "\n" + + "request_id\x18\x03 \x01(\tR\trequestId\"\xfd\x01\n" + "\x0eGeneratorState\x12J\n" + "\x13last_processed_time\x18\x03 \x01(\v2\x1a.google.protobuf.TimestampR\x11lastProcessedTime\x12J\n" + "\x13future_action_times\x18\x04 \x03(\v2\x1a.google.protobuf.TimestampR\x11futureActionTimes\x12S\n" + diff --git a/chasm/lib/scheduler/migration_chain_counterexample_test.go b/chasm/lib/scheduler/migration_chain_ownership_test.go similarity index 87% rename from chasm/lib/scheduler/migration_chain_counterexample_test.go rename to chasm/lib/scheduler/migration_chain_ownership_test.go index ed480e8a976..06c6fc595d9 100644 --- a/chasm/lib/scheduler/migration_chain_counterexample_test.go +++ b/chasm/lib/scheduler/migration_chain_ownership_test.go @@ -14,8 +14,7 @@ import ( "google.golang.org/grpc" ) -func TestMigrationCounterexampleRollbackChainOwnership(t *testing.T) { - requireMigrationCounterexamples(t) +func TestMigrationRollbackChainOwnership(t *testing.T) { for _, scenario := range []string{"owned_descendant", "foreign_chain", "first_run_removed"} { t.Run(scenario, func(t *testing.T) { e, handler, history := newRollbackScenario(t) @@ -40,10 +39,14 @@ func TestMigrationCounterexampleRollbackChainOwnership(t *testing.T) { }}, nil }).AnyTimes() _, err := executeRollbackTask(t, e, handler, e.engine) - if scenario == "owned_descendant" { + switch scenario { + case "owned_descendant": require.NoError(t, err) - } else { - require.Error(t, err) + case "foreign_chain": + require.ErrorAs(t, err, new(*serviceerror.AlreadyExists)) + default: + require.ErrorAs(t, err, new(*serviceerror.FailedPrecondition)) + require.ErrorContains(t, err, "is gone") } require.True(t, verified, "a descendant's start request does not establish ownership of the original migration") require.NoError(t, e.readScheduler(func(s *scheduler.Scheduler, _ chasm.Context) error { diff --git a/chasm/lib/scheduler/migration_handoff_test.go b/chasm/lib/scheduler/migration_handoff_test.go index c465148f431..096b643c7e5 100644 --- a/chasm/lib/scheduler/migration_handoff_test.go +++ b/chasm/lib/scheduler/migration_handoff_test.go @@ -14,6 +14,7 @@ import ( "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/api/historyservice/v1" "go.temporal.io/server/api/historyservicemock/v1" + persistencespb "go.temporal.io/server/api/persistence/v1" schedulespb "go.temporal.io/server/api/schedule/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/lib/scheduler" @@ -159,15 +160,96 @@ func TestMigrationScenario_RollbackRetryBoundaries(t *testing.T) { } } -func TestMigrationCounterexample_RollbackDestinationCollision(t *testing.T) { - requireMigrationCounterexamples(t) +func TestMigrationRollbackDestinationCollision(t *testing.T) { e, handler, historyClient := newRollbackScenario(t) historyClient.EXPECT().StartWorkflowExecution(gomock.Any(), gomock.Any()).Return(nil, serviceerror.NewWorkflowExecutionAlreadyStarted("unrelated scheduler owns this ID", "unrelated-create-request", "unrelated-run")) + historyClient.EXPECT().GetMutableState(gomock.Any(), gomock.Any()).Return( + &historyservice.GetMutableStateResponse{FirstExecutionRunId: "unrelated-run"}, nil) + historyClient.EXPECT().DescribeMutableState(gomock.Any(), gomock.Any()).Return( + &historyservice.DescribeMutableStateResponse{DatabaseMutableState: &persistencespb.WorkflowMutableState{ + ExecutionState: &persistencespb.WorkflowExecutionState{RunId: "unrelated-run", CreateRequestId: "unrelated-create-request"}, + }}, nil) _, err := executeRollbackTask(t, e, handler, e.engine) require.NoError(t, e.readScheduler(func(s *scheduler.Scheduler, _ chasm.Context) error { require.False(t, s.Closed, "rollback must retain the source and its acknowledged trigger when a different workflow owns the destination ID") return nil })) - require.Error(t, err, "an unrelated destination is a conflict, not a successful handoff") + require.ErrorAs(t, err, new(*serviceerror.AlreadyExists), "an unrelated destination is a conflict, not a successful handoff") +} + +func TestMigrationRollbackSendsStoredRequestID(t *testing.T) { + e, handler, historyClient := newRollbackScenario(t) + var requestIDs []string + historyClient.EXPECT().StartWorkflowExecution(gomock.Any(), gomock.Any()).Times(2).DoAndReturn( + func(_ context.Context, req *historyservice.StartWorkflowExecutionRequest, _ ...grpc.CallOption) (*historyservice.StartWorkflowExecutionResponse, error) { + requestIDs = append(requestIDs, req.StartRequest.RequestId) + if len(requestIDs) == 1 { + return nil, serviceerror.NewUnavailable("injected") + } + return &historyservice.StartWorkflowExecutionResponse{RunId: "destination-run"}, nil + }) + _, err := executeRollbackTask(t, e, handler, e.engine) + require.Error(t, err) + _, err = executeRollbackTask(t, e, handler, e.engine) + require.NoError(t, err) + require.Equal(t, []string{"rollback", "rollback"}, requestIDs) +} + +func TestMigrationRollbackAssignsRequestIDToLegacyMigration(t *testing.T) { + e, handler, historyClient := newRollbackScenario(t) + require.NoError(t, e.updateScheduler(func(s *scheduler.Scheduler, _ chasm.MutableContext) error { + s.WorkflowMigration.RequestId = "" + return nil + })) + var requestID string + historyClient.EXPECT().StartWorkflowExecution(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, req *historyservice.StartWorkflowExecutionRequest, _ ...grpc.CallOption) (*historyservice.StartWorkflowExecutionResponse, error) { + requestID = req.StartRequest.RequestId + return nil, serviceerror.NewUnavailable("injected") + }) + _, err := executeRollbackTask(t, e, handler, e.engine) + require.Error(t, err) + require.NotEmpty(t, requestID) + require.NoError(t, e.readScheduler(func(s *scheduler.Scheduler, _ chasm.Context) error { + require.Equal(t, requestID, s.WorkflowMigration.GetRequestId(), "the ID must be persisted before the first start attempt") + require.False(t, s.Closed) + return nil + })) +} + +func TestMigrationRollbackLegacyMigrationRejectsUnprovenDestination(t *testing.T) { + e, handler, historyClient := newRollbackScenario(t) + require.NoError(t, e.updateScheduler(func(s *scheduler.Scheduler, _ chasm.MutableContext) error { + s.WorkflowMigration.RequestId = "" + return nil + })) + // An attempt made before the upgrade may have started the workflow under a random ID. + historyClient.EXPECT().StartWorkflowExecution(gomock.Any(), gomock.Any()).Return(nil, + serviceerror.NewWorkflowExecutionAlreadyStarted("old attempt", "random-pre-upgrade-id", "old-run")) + historyClient.EXPECT().GetMutableState(gomock.Any(), gomock.Any()).Return( + &historyservice.GetMutableStateResponse{FirstExecutionRunId: "old-run"}, nil) + historyClient.EXPECT().DescribeMutableState(gomock.Any(), gomock.Any()).Return( + &historyservice.DescribeMutableStateResponse{DatabaseMutableState: &persistencespb.WorkflowMutableState{ + ExecutionState: &persistencespb.WorkflowExecutionState{RunId: "old-run", CreateRequestId: "random-pre-upgrade-id"}, + }}, nil) + _, err := executeRollbackTask(t, e, handler, e.engine) + require.ErrorAs(t, err, new(*serviceerror.AlreadyExists)) + require.NoError(t, e.readScheduler(func(s *scheduler.Scheduler, _ chasm.Context) error { + require.False(t, s.Closed) + return nil + })) +} + +func TestMigrateToWorkflowGeneratesRequestID(t *testing.T) { + s, ctx, _ := setupSchedulerForTest(t) + _, err := s.MigrateToWorkflow(ctx, &schedulerpb.MigrateToWorkflowRequest{}) + require.NoError(t, err) + require.NotEmpty(t, s.WorkflowMigration.GetRequestId()) + stored := s.WorkflowMigration.GetRequestId() + + // A repeated request does not replace the stored ID. + _, err = s.MigrateToWorkflow(ctx, &schedulerpb.MigrateToWorkflowRequest{RequestId: "other"}) + require.NoError(t, err) + require.Equal(t, stored, s.WorkflowMigration.GetRequestId()) } diff --git a/chasm/lib/scheduler/proto/v1/message.proto b/chasm/lib/scheduler/proto/v1/message.proto index 3bfbbaadc03..39c1fca8ef0 100644 --- a/chasm/lib/scheduler/proto/v1/message.proto +++ b/chasm/lib/scheduler/proto/v1/message.proto @@ -68,6 +68,12 @@ message WorkflowMigrationState { // The schedule's notes before migration was initiated. string pre_migration_notes = 2; + + // Request ID used for every StartWorkflowExecution attempt of the destination + // V1 workflow. It identifies the destination as created by this migration. + // Empty only on migrations persisted before this field existed; the migrate + // task assigns one before its first start attempt. + string request_id = 3; } // CHASM scheduler's Generator internal state. diff --git a/chasm/lib/scheduler/scheduler.go b/chasm/lib/scheduler/scheduler.go index e2b3956597b..7383ce637a4 100644 --- a/chasm/lib/scheduler/scheduler.go +++ b/chasm/lib/scheduler/scheduler.go @@ -7,6 +7,7 @@ import ( "slices" "time" + "github.com/google/uuid" commonpb "go.temporal.io/api/common/v1" enumspb "go.temporal.io/api/enums/v1" schedulepb "go.temporal.io/api/schedule/v1" @@ -894,10 +895,18 @@ func (s *Scheduler) MigrateToWorkflow( return nil, ErrMigrationActionsPending } + // The request ID is sent on every start attempt of the destination workflow, + // so a retry can recognize its own workflow. + requestID := req.GetRequestId() + if requestID == "" { + requestID = uuid.NewString() + } + // Save pre-migration paused state, mark migration as pending, then pause. s.WorkflowMigration = &schedulerpb.WorkflowMigrationState{ PreMigrationPaused: s.Schedule.State.Paused, PreMigrationNotes: s.Schedule.State.Notes, + RequestId: requestID, } s.Schedule.State.Paused = true s.Schedule.State.Notes = "paused for migration to workflow-backed scheduler" diff --git a/chasm/lib/scheduler/scheduler_migrate_task.go b/chasm/lib/scheduler/scheduler_migrate_task.go index 60376647ff4..748eceae923 100644 --- a/chasm/lib/scheduler/scheduler_migrate_task.go +++ b/chasm/lib/scheduler/scheduler_migrate_task.go @@ -12,6 +12,7 @@ import ( "go.temporal.io/api/serviceerror" taskqueuepb "go.temporal.io/api/taskqueue/v1" "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/server/api/historyservice/v1" schedulespb "go.temporal.io/server/api/schedule/v1" "go.temporal.io/server/chasm" schedulerpb "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" @@ -112,6 +113,7 @@ func (h *SchedulerMigrateToWorkflowTaskHandler) Execute( searchAttributes map[string]*commonpb.Payload memo map[string]*commonpb.Payload now time.Time + requestID string } var result readResult @@ -163,6 +165,7 @@ func (h *SchedulerMigrateToWorkflowTaskHandler) Execute( searchAttributes: searchAttributes, memo: memo, now: now, + requestID: schedulerState.GetWorkflowMigration().GetRequestId(), } return struct{}{}, nil }, @@ -172,6 +175,17 @@ func (h *SchedulerMigrateToWorkflowTaskHandler) Execute( return fmt.Errorf("failed to read scheduler state: %w", err) } + if result.requestID == "" { + // The migration was persisted before request IDs existed, so an earlier + // attempt may have started a workflow under a random ID. Assign an ID + // now; if such a workflow exists, the ownership check below rejects it + // rather than accepting it unproven. + result.requestID, err = h.assignRequestID(ctx, schedulerRef) + if err != nil { + return err + } + } + logger = log.With( h.baseLogger, tag.WorkflowNamespace(result.namespace), @@ -209,7 +223,7 @@ func (h *SchedulerMigrateToWorkflowTaskHandler) Execute( } workflowID := legacyscheduler.WorkflowIDPrefix + result.scheduleID startReq := &workflowservice.StartWorkflowExecutionRequest{ - RequestId: uuid.NewString(), + RequestId: result.requestID, Namespace: result.namespace, WorkflowId: workflowID, WorkflowType: &commonpb.WorkflowType{Name: legacyscheduler.WorkflowType}, @@ -228,10 +242,17 @@ func (h *SchedulerMigrateToWorkflowTaskHandler) Execute( common.CreateHistoryStartWorkflowRequest(result.namespaceID, startReq, nil, nil, result.now), ) if err != nil { - // Treat already-started as success for idempotency. - if _, ok := errors.AsType[*serviceerror.WorkflowExecutionAlreadyStarted](err); !ok { + alreadyStarted, ok := errors.AsType[*serviceerror.WorkflowExecutionAlreadyStarted](err) + if !ok { return fmt.Errorf("failed to start V1 scheduler workflow: %w", err) } + // Only a workflow created by this migration is an idempotent success. + if alreadyStarted.StartRequestId != result.requestID { + err = h.verifyDestinationOwner(ctx, result.namespaceID, workflowID, alreadyStarted.RunId, result.requestID) + if err != nil { + return err + } + } } // Mark the CHASM scheduler as closed now that the V1 workflow is running. @@ -251,3 +272,67 @@ func (h *SchedulerMigrateToWorkflowTaskHandler) Execute( return nil } + +// assignRequestID persists a request ID for a migration that has none and +// returns the stored ID. It keeps an ID that a concurrent attempt already stored. +func (h *SchedulerMigrateToWorkflowTaskHandler) assignRequestID( + ctx context.Context, + schedulerRef chasm.ComponentRef, +) (string, error) { + requestID, _, err := chasm.UpdateComponent( + ctx, + schedulerRef, + func(s *Scheduler, _ chasm.MutableContext, _ any) (string, error) { + if s.Closed || s.WorkflowMigration == nil { + return "", ErrMigrationPending + } + if s.WorkflowMigration.RequestId == "" { + s.WorkflowMigration.RequestId = uuid.NewString() + } + return s.WorkflowMigration.RequestId, nil + }, + nil, + ) + if err != nil { + return "", fmt.Errorf("failed to assign migration request ID: %w", err) + } + return requestID, nil +} + +// verifyDestinationOwner checks that the V1 workflow chain containing runID was +// created by this migration. The current run's start request ID differs from the +// migration's after continue-as-new, so the check compares the chain's first +// run against the migration's request ID. It returns an error unless that +// ownership is proven. +func (h *SchedulerMigrateToWorkflowTaskHandler) verifyDestinationOwner( + ctx context.Context, + namespaceID, workflowID, runID, requestID string, +) error { + current, err := h.historyClient.GetMutableState(ctx, &historyservice.GetMutableStateRequest{ + NamespaceId: namespaceID, + Execution: &commonpb.WorkflowExecution{WorkflowId: workflowID, RunId: runID}, + }) + if err != nil { + return fmt.Errorf("failed to resolve V1 scheduler workflow chain: %w", err) + } + firstRunID := current.GetFirstExecutionRunId() + if firstRunID == "" { + return serviceerror.NewFailedPreconditionf("first run of V1 scheduler workflow %q is unknown", workflowID) + } + first, err := h.historyClient.DescribeMutableState(ctx, &historyservice.DescribeMutableStateRequest{ + NamespaceId: namespaceID, + Execution: &commonpb.WorkflowExecution{WorkflowId: workflowID, RunId: firstRunID}, + }) + if _, ok := errors.AsType[*serviceerror.NotFound](err); ok { + // Without the first run, ownership cannot be proven. + return serviceerror.NewFailedPreconditionf("first run %q of V1 scheduler workflow %q is gone; cannot verify it was created by this migration", firstRunID, workflowID) + } + if err != nil { + return fmt.Errorf("failed to verify V1 scheduler workflow owner: %w", err) + } + state := first.GetDatabaseMutableState().GetExecutionState() + if state.GetRunId() != firstRunID || state.GetCreateRequestId() != requestID { + return serviceerror.NewAlreadyExistsf("V1 scheduler workflow %q was not created by this migration", workflowID) + } + return nil +} diff --git a/chasm/lib/scheduler/scheduler_migrate_test.go b/chasm/lib/scheduler/scheduler_migrate_test.go index 109c40322b1..4d4187b1f5a 100644 --- a/chasm/lib/scheduler/scheduler_migrate_test.go +++ b/chasm/lib/scheduler/scheduler_migrate_test.go @@ -100,7 +100,7 @@ func TestMigrateToWorkflowTask_DoesNotPropagateDisabledTimeSkipping(t *testing.T }, } } - env.Scheduler.WorkflowMigration = &schedulerpb.WorkflowMigrationState{} + env.Scheduler.WorkflowMigration = &schedulerpb.WorkflowMigrationState{RequestId: "migration-request"} readCtx := env.ReadContext() env.ExpectReadComponent(readCtx, env.Scheduler) diff --git a/service/frontend/admin_handler.go b/service/frontend/admin_handler.go index 20ee940bc6d..c76634d2d62 100644 --- a/service/frontend/admin_handler.go +++ b/service/frontend/admin_handler.go @@ -2543,6 +2543,11 @@ func (adh *AdminHandler) migrateScheduleToWorkflow( request *adminservice.MigrateScheduleRequest, namespaceID string, ) (*adminservice.MigrateScheduleResponse, error) { + // The scheduler stores this ID and uses it to start the V1 workflow. + migrationRequestID := request.GetRequestId() + if migrationRequestID == "" { + migrationRequestID = uuid.NewString() + } workflowID := scheduler.WorkflowIDPrefix + request.GetScheduleId() descResp, err := adh.historyClient.DescribeWorkflowExecution(ctx, &historyservice.DescribeWorkflowExecutionRequest{ NamespaceId: namespaceID, @@ -2587,7 +2592,7 @@ func (adh *AdminHandler) migrateScheduleToWorkflow( NamespaceId: namespaceID, ScheduleId: request.GetScheduleId(), Identity: request.GetIdentity(), - RequestId: request.GetRequestId(), + RequestId: migrationRequestID, }, ) if err != nil { diff --git a/service/frontend/admin_handler_test.go b/service/frontend/admin_handler_test.go index 929f4e11da1..a69491c736f 100644 --- a/service/frontend/admin_handler_test.go +++ b/service/frontend/admin_handler_test.go @@ -2377,6 +2377,29 @@ func (f *fakeSchedulerClient) MigrateToWorkflow(ctx context.Context, req *schedu return f.migrateToWorkflowFn(ctx, req) } +func (s *adminHandlerSuite) TestMigrateScheduleToWorkflowGeneratesRequestID() { + s.mockNamespaceCache.EXPECT().GetNamespaceID(s.namespace).Return(s.namespaceID, nil) + s.mockHistoryClient.EXPECT().DescribeWorkflowExecution(gomock.Any(), gomock.Any()).Return( + nil, serviceerror.NewNotFound("workflow not found")) + + var capturedReq *schedulerpb.MigrateToWorkflowRequest + s.handler.schedulerClient = &fakeSchedulerClient{ + migrateToWorkflowFn: func(_ context.Context, req *schedulerpb.MigrateToWorkflowRequest) (*schedulerpb.MigrateToWorkflowResponse, error) { + capturedReq = req + return &schedulerpb.MigrateToWorkflowResponse{}, nil + }, + } + + _, err := s.handler.MigrateSchedule(context.Background(), &adminservice.MigrateScheduleRequest{ + Namespace: s.namespace.String(), + ScheduleId: "test-schedule", + Target: adminservice.MigrateScheduleRequest_SCHEDULER_TARGET_WORKFLOW, + Identity: "test-identity", + }) + s.NoError(err) + s.NoError(uuid.Validate(capturedReq.GetRequestId())) +} + func (s *adminHandlerSuite) TestMigrateScheduleToWorkflow() { s.mockNamespaceCache.EXPECT().GetNamespaceID(s.namespace).Return(s.namespaceID, nil) s.mockHistoryClient.EXPECT().DescribeWorkflowExecution(gomock.Any(), &historyservice.DescribeWorkflowExecutionRequest{ From 818f7259e8bb2b2110d0e4ff32a421baf49250b5 Mon Sep 17 00:00:00 2001 From: chaptersix <13949480+chaptersix@users.noreply.github.com> Date: Wed, 11 Nov 2026 16:39:07 -0600 Subject: [PATCH 08/22] Hold the overlap slot for selected starts awaiting their RunId processBuffer computed isRunning only from starts with a RunId, so a start selected for execution (Attempt > 0) but not yet recorded did not block a second ProcessBuffer pass from selecting another start under single-action overlap policies. Count tracked, uncompleted starts with Attempt > 0 as running. Because a dropped selected start now defers later starts without producing a completion, and a recorded RunId makes it cancelable, re-enable deferred starts in recordExecuteResult when a start is dropped or newly started. Co-Authored-By: Claude Fable 5.2 --- .../scheduler/generator_killswitch_test.go | 2 + chasm/lib/scheduler/handler_test.go | 5 +- chasm/lib/scheduler/invoker.go | 23 ++++-- chasm/lib/scheduler/invoker_tasks.go | 10 ++- .../schedulertest/check_mutations.py | 6 +- .../scheduler/schedulertest/inflight_test.go | 82 ++++++++++++++++--- 6 files changed, 108 insertions(+), 20 deletions(-) diff --git a/chasm/lib/scheduler/generator_killswitch_test.go b/chasm/lib/scheduler/generator_killswitch_test.go index 9fbd7fceebd..dcafd1fbed9 100644 --- a/chasm/lib/scheduler/generator_killswitch_test.go +++ b/chasm/lib/scheduler/generator_killswitch_test.go @@ -5,6 +5,7 @@ import ( "time" "github.com/stretchr/testify/require" + enumspb "go.temporal.io/api/enums/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/lib/scheduler" "go.temporal.io/server/common/clock" @@ -42,6 +43,7 @@ func TestGeneratorJitterFixKillswitch(t *testing.T) { schedule := defaultSchedule() schedule.Spec.Interval[0].Interval = durationpb.New(15 * time.Minute) schedule.Spec.Jitter = durationpb.New(10 * time.Minute) + schedule.Policies.OverlapPolicy = enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ALL env := newSchedulerTestEngine(t, schedule, withEngineTimeSource(ts), withEngineSpecProcessor(processor)) // Change the setting after creation to exercise existing schedules. diff --git a/chasm/lib/scheduler/handler_test.go b/chasm/lib/scheduler/handler_test.go index 27bab78c1c7..a03ad14c5dc 100644 --- a/chasm/lib/scheduler/handler_test.go +++ b/chasm/lib/scheduler/handler_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/stretchr/testify/require" + enumspb "go.temporal.io/api/enums/v1" schedulepb "go.temporal.io/api/schedule/v1" "go.temporal.io/api/serviceerror" "go.temporal.io/api/workflowservice/v1" @@ -196,7 +197,9 @@ func TestHandler_MigrateToWorkflow_Sentinel(t *testing.T) { } func TestHandler_PatchSchedule_DedupsRequestID(t *testing.T) { - e := newSchedulerTestEngine(t, defaultSchedule()) + schedule := defaultSchedule() + schedule.Policies.OverlapPolicy = enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ALL + e := newSchedulerTestEngine(t, schedule) h := scheduler.NewTestHandler(e.logger) patch := func(requestID string) { _, err := h.PatchSchedule(e.engineCtx, &schedulerpb.PatchScheduleRequest{ diff --git a/chasm/lib/scheduler/invoker.go b/chasm/lib/scheduler/invoker.go index e649388c6a6..45ea86b4011 100644 --- a/chasm/lib/scheduler/invoker.go +++ b/chasm/lib/scheduler/invoker.go @@ -223,6 +223,7 @@ func (i *Invoker) recordExecuteResult( } return failed }) + dropped := removedStarts > 0 i.CancelWorkflows = slices.DeleteFunc(i.GetCancelWorkflows(), func(we *commonpb.WorkflowExecution) bool { canceled := canceled[we.RunId] if canceled { @@ -269,6 +270,12 @@ func (i *Invoker) recordExecuteResult( _, remove := startedUntracked[start.GetRequestId()] return remove }) + // A selected start holds the overlap slot until its RunId is recorded. Dropping it + // produces no completion, and recording its RunId makes it cancelable, so either + // way deferred starts must be re-evaluated against the new state. + if dropped || newlyStarted > 0 { + i.reenableDeferredStarts() + } i.getOrCreateEventLog(ctx).LogEvent(ctx, fmt.Sprintf("recordExecuteResult kicked off %d starts, removed %d starts, retried %d starts", newlyStarted, @@ -330,11 +337,7 @@ func (i *Invoker) recordCompletedAction( // Re-enable deferred starts (Attempt == -1) so they can be re-processed by // ProcessBuffer now that a workflow has completed. This allows the overlap // policy to be re-evaluated. - for _, start := range i.BufferedStarts { - if start.Attempt == -1 { - start.Attempt = 0 - } - } + i.reenableDeferredStarts() // Update DesiredTime on the first pending start for metrics. DesiredTime is used // to drive action latency between buffered starts (the time it takes between @@ -359,6 +362,16 @@ func (i *Invoker) recordCompletedAction( return } +// reenableDeferredStarts returns deferred starts (Attempt == -1) to the pending +// state so the next ProcessBuffer pass re-evaluates their overlap policy. +func (i *Invoker) reenableDeferredStarts() { + for _, start := range i.BufferedStarts { + if start.Attempt == -1 { + start.Attempt = 0 + } + } +} + // addTasks adds both ProcessBuffer and Execute tasks as needed. It should be // called when completing processing/executing tasks, to drive backoff/retry. func (i *Invoker) addTasks(ctx chasm.MutableContext) { diff --git a/chasm/lib/scheduler/invoker_tasks.go b/chasm/lib/scheduler/invoker_tasks.go index d8853860d17..8dd7c554ffa 100644 --- a/chasm/lib/scheduler/invoker_tasks.go +++ b/chasm/lib/scheduler/invoker_tasks.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "slices" "sync" "time" @@ -537,7 +538,14 @@ func (h *InvokerProcessBufferTaskHandler) processBuffer( scheduler *Scheduler, ) (result processBufferResult) { runningWorkflows := invoker.runningWorkflowExecutions() - isRunning := len(runningWorkflows) > 0 + // A start that was selected for execution but has no recorded RunId yet still + // holds the overlap slot, otherwise a second pass could select another start + // before the first StartWorkflowExecution response is recorded. + isRunning := len(runningWorkflows) > 0 || + slices.ContainsFunc(invoker.GetBufferedStarts(), func(start *schedulespb.BufferedStart) bool { + return start.GetAttempt() > 0 && start.GetCompleted() == nil && + internal.TracksCompletionResult(start.GetOverlapPolicy()) + }) result.missedCatchupByActionRunning = make(map[bool]int64) // Processing ignores starts that are already executing or backing off. An existing diff --git a/chasm/lib/scheduler/schedulertest/check_mutations.py b/chasm/lib/scheduler/schedulertest/check_mutations.py index 9a6cd626e05..e62c6e868f9 100644 --- a/chasm/lib/scheduler/schedulertest/check_mutations.py +++ b/chasm/lib/scheduler/schedulertest/check_mutations.py @@ -17,6 +17,10 @@ '\t// A discarded selection cannot produce', '\t// Re-arm tasks if this call changed state'), ("backfiller_tasks.go", "capacity-progress", "./chasm/lib/scheduler", "TestBackfillCapacityProgress", None, None), + ("invoker_tasks.go", "selected-start-reservation", "./chasm/lib/scheduler/schedulertest", + "TestSelectedStartReservesOverlapSlot", + "start.GetAttempt() > 0 && start.GetCompleted() == nil &&\n\t\t\t\t", + "internal.TracksCompletionResult(start.GetOverlapPolicy())\n\t\t})"), ] for filename, name, package, test, begin, end in mutations: source = root / "chasm/lib/scheduler" / filename @@ -24,7 +28,7 @@ if begin is not None: first = text.index(begin) last = text.index(end, first) - text = text[:first] + text[last:] + text = text[:first] + ("false && " if name == "selected-start-reservation" else "") + text[last:] else: original = "return min(available, max(1, available/backfillerCount))" assert text.count(original) == 1 diff --git a/chasm/lib/scheduler/schedulertest/inflight_test.go b/chasm/lib/scheduler/schedulertest/inflight_test.go index ae1c8cc1aec..5841ef88bf9 100644 --- a/chasm/lib/scheduler/schedulertest/inflight_test.go +++ b/chasm/lib/scheduler/schedulertest/inflight_test.go @@ -6,6 +6,7 @@ import ( "github.com/stretchr/testify/require" enumspb "go.temporal.io/api/enums/v1" + "go.temporal.io/api/serviceerror" "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/lib/scheduler" ) @@ -41,9 +42,50 @@ func TestStartResultSurvivesBufferReordering(t *testing.T) { require.Equal(t, acceptedSecond, snap.RecordedStarts[second.Request.GetRequestId()]) } -// This detector documents the reservation gap from #59/#62 without importing -// their experimental action schema into the lifecycle driver. -func TestSelectedStartReservationCounterexample(t *testing.T) { +// selectedStarts counts starts chosen for execution whose workflow has not completed. +func selectedStarts(d *Driver) (selected int) { + d.ReadScheduler(func(s *scheduler.Scheduler, ctx chasm.Context) { + for _, start := range s.Invoker.Get(ctx).BufferedStarts { + if start.GetAttempt() > 0 && start.GetCompleted() == nil { + selected++ + } + } + }) + return selected +} + +func TestSelectedStartReservesOverlapSlot(t *testing.T) { + t.Parallel() + for _, policy := range []enumspb.ScheduleOverlapPolicy{ + enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ALL, + enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ONE, + } { + t.Run(policy.String(), func(t *testing.T) { + t.Parallel() + schedule := DefaultSchedule() + schedule.State.Paused = true + schedule.Policies.OverlapPolicy = policy + d := NewDriver(t, WithSchedule(schedule), WithControlledStarts()) + require.NoError(t, d.TriggerImmediately()) + require.True(t, d.BeginSideEffect()) + call := d.NextStart() + accepted, err := call.Accept() + require.NoError(t, err) + require.NoError(t, d.TriggerImmediately()) + require.Equal(t, 1, selectedStarts(d), "second start selected before the first acknowledgement") + call.Respond(accepted, nil) + d.FinishSideEffect() + require.Equal(t, 1, selectedStarts(d)) + + require.NoError(t, d.DeliverCompletion(call.Request.GetRequestId())) + require.True(t, d.StepPure()) + require.Equal(t, 1, selectedStarts(d), "second start not selected after the first completed") + require.Empty(t, CheckInvariants(nil, d.Snapshot())) + }) + } +} + +func TestFailedSelectedStartReleasesOverlapSlot(t *testing.T) { t.Parallel() schedule := DefaultSchedule() schedule.State.Paused = true @@ -52,20 +94,36 @@ func TestSelectedStartReservationCounterexample(t *testing.T) { require.NoError(t, d.TriggerImmediately()) require.True(t, d.BeginSideEffect()) call := d.NextStart() + require.NoError(t, d.TriggerImmediately()) + require.Equal(t, 1, selectedStarts(d)) + call.Respond(ExternalStart{}, serviceerror.NewInvalidArgument("rejected")) + d.FinishSideEffect() + + require.True(t, d.StepPure()) + require.Equal(t, 1, selectedStarts(d), "second start not selected after the first was dropped") + require.Empty(t, CheckInvariants(nil, d.Snapshot())) +} + +func TestSelectedStartReservationStillTerminatesOther(t *testing.T) { + t.Parallel() + schedule := DefaultSchedule() + schedule.State.Paused = true + schedule.Policies.OverlapPolicy = enumspb.SCHEDULE_OVERLAP_POLICY_TERMINATE_OTHER + d := NewDriver(t, WithSchedule(schedule), WithControlledStarts()) + require.NoError(t, d.TriggerImmediately()) + require.True(t, d.BeginSideEffect()) + call := d.NextStart() accepted, err := call.Accept() require.NoError(t, err) require.NoError(t, d.TriggerImmediately()) - selected := 0 - d.ReadScheduler(func(s *scheduler.Scheduler, ctx chasm.Context) { - for _, start := range s.Invoker.Get(ctx).BufferedStarts { - if start.GetAttempt() > 0 && start.GetCompleted() == nil { - selected++ - } - } - }) call.Respond(accepted, nil) d.FinishSideEffect() - require.Equal(t, 2, selected, "known counterexample: two BUFFER_ALL starts selected before the first acknowledgement; replace with <=1 when fixed") + require.True(t, d.StepPure()) + terminates := 0 + d.ReadScheduler(func(s *scheduler.Scheduler, ctx chasm.Context) { + terminates = len(s.Invoker.Get(ctx).TerminateWorkflows) + }) + require.Equal(t, 1, terminates, "deferred start never terminated the running workflow") } // Migration is refused while a start is ready to execute, since V1 cannot resume From a4e1b5cbf106991f0f6c8af7b65e94bcd9828563 Mon Sep 17 00:00:00 2001 From: chaptersix <13949480+chaptersix@users.noreply.github.com> Date: Wed, 11 Nov 2026 16:42:37 -0600 Subject: [PATCH 09/22] Document ported scheduler fixes and unify remaining counterexamples Record the production fixes and their default-run regressions in the coverage document, along with the dedicated property-test job. Name the four still-gated migration cases with one TestMigrationCounterexample prefix and state why each needs a handoff protocol. Drop the gate helper that no longer has callers in the scheduler package. Co-Authored-By: Claude Fable 5.2 --- chasm/lib/scheduler/migration_handoff_test.go | 8 -- .../chasm-schedulertest-coverage-plan.md | 86 ++++++++++++++----- .../frontend/migration_delete_race_test.go | 2 +- .../scheduler/migration_scenarios_test.go | 2 +- 4 files changed, 66 insertions(+), 32 deletions(-) diff --git a/chasm/lib/scheduler/migration_handoff_test.go b/chasm/lib/scheduler/migration_handoff_test.go index 096b643c7e5..b33312c4ae9 100644 --- a/chasm/lib/scheduler/migration_handoff_test.go +++ b/chasm/lib/scheduler/migration_handoff_test.go @@ -2,7 +2,6 @@ package scheduler_test import ( "context" - "os" "testing" "time" @@ -35,13 +34,6 @@ type migrationCloseFaultEngine struct { armed bool } -func requireMigrationCounterexamples(t *testing.T) { - t.Helper() - if os.Getenv("TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES") != "1" { - t.Skip("known migration counterexample; set TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES=1 to run") - } -} - func (e *migrationCloseFaultEngine) UpdateComponent( ctx context.Context, ref chasm.ComponentRef, update func(chasm.MutableContext, chasm.Component) error, opts ...chasm.TransitionOption, ) ([]byte, error) { diff --git a/docs/architecture/chasm-schedulertest-coverage-plan.md b/docs/architecture/chasm-schedulertest-coverage-plan.md index 3a8b337126e..35554f9b1e9 100644 --- a/docs/architecture/chasm-schedulertest-coverage-plan.md +++ b/docs/architecture/chasm-schedulertest-coverage-plan.md @@ -17,13 +17,13 @@ authorship. The following signed commit contains the additions described here. | [#35](https://github.com/chaptersix/temporal/pull/35) | `task_validity_engine_test.go` ports generator, idle, callback-repair and migration validation cases. `schedulertest/task_validity_test.go` distinguishes stale physical deliveries from valid current work, checks retry equality/rearming, and detects an advertised idle deadline with no valid logical task. | | [#41](https://github.com/chaptersix/temporal/pull/41) | `TestDeferredBufferOneProperty` adapts the occupancy/accounting property to actual handlers. Existing `TestCompletedHistoryDoesNotConsumeBackfillCapacity`, capacity arithmetic tests and duplicate-result tests remain part of the required package run; no experimental planner refactor is imported. | | [#51](https://github.com/chaptersix/temporal/pull/51), [#52](https://github.com/chaptersix/temporal/pull/52) | Ports 450/451/1000-range finite-drain tests and the positive-capacity admission fix. The test supplies capacity by consuming each batch; it checks unique identities, bounds and termination, not cross-schedule fairness. | -| [#42](https://github.com/chaptersix/temporal/pull/42), [#43](https://github.com/chaptersix/temporal/pull/43) | Native exclusive-cursor control plus opt-in migrated cursor/epoch and callback-reconciliation counterexamples in `migration_backfill_test.go`. Existing stamped/legacy task tests remain required. The migration schema/progress-bit fix is not imported. | -| [#53](https://github.com/chaptersix/temporal/pull/53), [#55](https://github.com/chaptersix/temporal/pull/55) | Native phase controls, conversion evidence and opt-in rollback phase counterexamples. The driver reports execution/retry/cancel/terminate obligations made unexecutable by migration; it no longer exempts every migrating state from its ledger. The rollback production gate is not imported. | -| [#59](https://github.com/chaptersix/temporal/pull/59), [#62](https://github.com/chaptersix/temporal/pull/62) | Controlled acceptance/response events, buffer reordering during an outstanding RPC, repeated result reconciliation, and a directed selected-start reservation detector. Workflow scenarios are adapted without importing experimental activity APIs or action schemas. | +| [#42](https://github.com/chaptersix/temporal/pull/42), [#43](https://github.com/chaptersix/temporal/pull/43) | Native exclusive-cursor control plus migrated cursor/epoch and callback-reconciliation regressions in `migration_backfill_test.go`, with the cursor and reconciliation-ordering fixes. Watermark presence replaces the separate progress bit, so no schema change is imported. | +| [#53](https://github.com/chaptersix/temporal/pull/53), [#55](https://github.com/chaptersix/temporal/pull/55) | Native phase controls, conversion evidence, rollback phase regressions and the rollback production gate. The driver reports execution/retry/cancel/terminate obligations made unexecutable by migration; it no longer exempts every migrating state from its ledger. | +| [#59](https://github.com/chaptersix/temporal/pull/59), [#62](https://github.com/chaptersix/temporal/pull/62) | Controlled acceptance/response events, buffer reordering during an outstanding RPC, repeated result reconciliation, and directed selected-start reservation regressions with a minimal reservation fix. Workflow scenarios are adapted without importing experimental activity APIs or action schemas. | | [#70](https://github.com/chaptersix/temporal/pull/70) | Duplicate callbacks before/after local start recording for ALLOW_ALL, SKIP and BUFFER_ALL, plus the early ALLOW_ALL callback guard. This branch retains callback attachment; the separate rollout-dependent callback-omission change is not imported. | | [#44](https://github.com/chaptersix/temporal/pull/44), [#45](https://github.com/chaptersix/temporal/pull/45) | `migration_handoff_test.go` and the V1 SDK workflow scenarios exercise lost responses, source-close boundaries, destination collisions and signal admission. | | [#56](https://github.com/chaptersix/temporal/pull/56) | `service/frontend/migration_delete_race_test.go` includes the native control and an opt-in held-create/delete counterexample. V1 termination is mocked. | -| [#57](https://github.com/chaptersix/temporal/pull/57) | `migration_chain_counterexample_test.go` adapts owned-descendant, foreign-chain and expired-first-run ownership checks to this baseline. Receipt persistence and the production ownership protocol are not imported. | +| [#57](https://github.com/chaptersix/temporal/pull/57) | `migration_chain_ownership_test.go` adapts owned-descendant, foreign-chain and expired-first-run ownership checks to this baseline, with the persisted request ID and first-run ownership check. Receipt and start-pending persistence are not imported because they make a pre-commit Unavailable permanent. | ## Driver changes @@ -54,14 +54,20 @@ Go `go1.27.3 darwin/arm64` and Rapid `v1.3.0`. ```sh CGO_ENABLED=0 go test -tags test_dep ./chasm/chasmtest/... ./chasm/lib/scheduler/... ./service/frontend ./service/worker/scheduler -count=1 -CGO_ENABLED=0 go test -tags test_dep ./chasm/lib/scheduler/schedulertest -run TestSchedulerProperties -rapid.checks=1000 -count=1 +CGO_ENABLED=0 make property-test TEST_RACE_FLAG=off python3 chasm/lib/scheduler/schedulertest/check_mutations.py CGO_ENABLED=0 make lint-code-fast GOLANGCI_LINT_BASE_REV=03ecd1d228d2b8d9d433da55785afc9aeeead918 GOLANGCI_LINT_FIX=false ``` -The package tests, 1,000 Rapid cases, all three mutation checks and fast lint pass. Mutations -remove the early callback guard, deferred rearming, and positive-capacity -admission independently; each produces assertion failures in its targeted tests. +The package tests, 5,000 Rapid cases, all four mutation checks and fast lint pass. Mutations +remove the early callback guard, deferred rearming, positive-capacity admission +and the selected-start reservation independently; each produces assertion +failures in its targeted tests. + +`make property-test` is also a dedicated CI job. The unit-test job retries +failures, and each retry draws a new Rapid seed, so a rare violation could be +retried away there. The dedicated job runs more cases with no retries and +uploads the minimized failfiles from `testdata/rapid`. The original early callback run failed on `accepted-start-lost`, with ActionCount still zero after external acceptance. The fix preserves exactly one counted start. @@ -76,26 +82,62 @@ The seeded replay is: CGO_ENABLED=0 go test -tags test_dep ./chasm/lib/scheduler/schedulertest -run TestSchedulerProperties -rapid.seed=7487548259739501542 -count=1 ``` +## Production fixes made for reproduced bugs + +Each fix has a default-run regression; none is gated. + +- **Backfill cursor replay.** A migrated backfiller carries the V1 exclusive + cursor as its watermark. Watermark presence, not value, marks recorded + progress, so an epoch cursor is honoured. + `TestMigrationBackfill_MigratedResumesAfterWatermark`. +- **Backfill before callback reconciliation.** Migrated backfillers get their + first task only after `SchedulerCallbacksTask` reconciles imported running + state. `TestMigrationBackfill_WaitsForCallbackReconciliation`. +- **Fresh backfill rollback boundary.** Rollback shifts an unstarted range back + 1ms to keep its inclusive first point on V1's exclusive cursor. + `TestMigrationScenario_RollbackFreshBackfill`. +- **Rollback execution phases.** `MigrateToWorkflow` and the migrate task return + the retryable `ErrMigrationActionsPending` while a selected start is + unrecorded or cancel/terminate requests are queued, because V1 state cannot + represent them. `TestRollbackPhaseGate`, `TestRollbackPhaseTransferable`. +- **Rollback destination ownership.** `WorkflowMigrationState.request_id` is + persisted and sent on every start attempt. An already-started destination is + accepted only if its start request, or the create request of its chain's first + run, matches. A foreign chain or a missing first run leaves the source open. + A migration persisted without an ID is assigned one before its first attempt; + if an earlier binary already created its destination under a random ID, the + task fails closed and needs an operator. + `TestMigrationRollbackDestinationCollision`, `TestMigrationRollbackChainOwnership`. +- **Selected-start reservation.** A selected but unacknowledged start holds the + overlap slot. Deferred starts are reconsidered when a start records its run or + is dropped, so a failed start cannot strand the buffer and TERMINATE_OTHER and + CANCEL_OTHER still act. `TestSelectedStartReservesOverlapSlot`, + `TestFailedSelectedStartReleasesOverlapSlot`, + `TestSelectedStartReservationStillTerminatesOther`. + ## Unresolved counterexamples and limits -The migration cases are evidence, not fixes. Enable their intended correctness -assertions explicitly; this command is expected to fail on this branch: +Four migration cases remain evidence, not fixes. Enable their intended +correctness assertions explicitly; this command is expected to fail: ```sh -TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES=1 CGO_ENABLED=0 go test -tags test_dep ./chasm/lib/scheduler ./service/frontend ./service/worker/scheduler -run 'TestMigrationCounterexample|TestMigrationBackfill_(Migrated|Waits)|TestMigrationDeleteCounterexample|TestRollbackPhaseCounterexample' -count=1 +TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES=1 CGO_ENABLED=0 go test -tags test_dep ./service/frontend ./service/worker/scheduler -run TestMigrationCounterexample -count=1 ``` -Observed failures include non-epoch migrated cursor replay, callback-barrier -ordering, rollback execution phases, foreign ownership, source signals during -handoff, ambiguous committed create responses, fresh rollback range boundaries, -delete/create ordering, and descendant ownership verification. The epoch case is -included but does not fail in this fixture; that is not a proof of epoch-state -representation. None of these tests establishes namespace failover safety. - -`TestSelectedStartReservationCounterexample` is an always-run detector asserting -the observed two selected BUFFER_ALL starts before the first acknowledgement. -It records the remaining reservation gap; replace that characterization with a -one-slot correctness assertion when the production fix is ported. +- `TestMigrationCounterexample_TriggerDuringSnapshot`: a signal accepted while + the migrate activity is pending is dropped with the old snapshot. +- `TestMigrationCounterexample_ForwardDestinationCollision`: the forward + activity accepts an unrelated existing CHASM schedule as its destination. +- `TestMigrationCounterexample_LostCreateResponse`: after an ambiguous create, + the V1 source keeps executing transferred work. +- `TestMigrationCounterexample_DeleteDuringCreate`: a create held across + `DeleteSchedule` commits an active scheduler afterwards. + +All four need a handoff protocol that fences source admission and identifies the +migration at the destination; no local production change closes them. The +lost-response case also asserts behaviour at existing workflow versions, so its +fix must be version-gated and its test adjusted. None of these tests establishes +namespace failover safety. The default driver remains a bounded, single-schedule environment. Request IDs are unique per generated occurrence. The external ledger retains deduplication diff --git a/service/frontend/migration_delete_race_test.go b/service/frontend/migration_delete_race_test.go index 86751e03ce1..4efd279255c 100644 --- a/service/frontend/migration_delete_race_test.go +++ b/service/frontend/migration_delete_race_test.go @@ -42,7 +42,7 @@ func (c *migrationDeleteRaceClient) DeleteSchedule(_ context.Context, req *sched } func TestMigrationDeleteNativeControl(t *testing.T) { testMigrationDeleteRace(t, false) } -func TestMigrationDeleteCounterexample(t *testing.T) { +func TestMigrationCounterexample_DeleteDuringCreate(t *testing.T) { if os.Getenv("TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES") != "1" { t.Skip("known migration counterexample; set TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES=1 to run") } diff --git a/service/worker/scheduler/migration_scenarios_test.go b/service/worker/scheduler/migration_scenarios_test.go index 04549ab5116..9f8b92f4d66 100644 --- a/service/worker/scheduler/migration_scenarios_test.go +++ b/service/worker/scheduler/migration_scenarios_test.go @@ -150,7 +150,7 @@ func TestMigrationScenario_NativeBackfillIncludesStart(t *testing.T) { checkRollbackBackfillStart(t, false) } -func TestMigrationCounterexample_RollbackFreshBackfill(t *testing.T) { +func TestMigrationScenario_RollbackFreshBackfill(t *testing.T) { checkRollbackBackfillStart(t, true) } From 0bf2ff75323f39ac0c5672687df151409b4e9c37 Mon Sep 17 00:00:00 2001 From: chaptersix <13949480+chaptersix@users.noreply.github.com> Date: Wed, 11 Nov 2026 17:59:08 -0600 Subject: [PATCH 10/22] Record the completed race-detector run in the coverage document Co-Authored-By: Claude Fable 5.2 --- docs/architecture/chasm-schedulertest-coverage-plan.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/architecture/chasm-schedulertest-coverage-plan.md b/docs/architecture/chasm-schedulertest-coverage-plan.md index 35554f9b1e9..477d8eb8984 100644 --- a/docs/architecture/chasm-schedulertest-coverage-plan.md +++ b/docs/architecture/chasm-schedulertest-coverage-plan.md @@ -153,6 +153,8 @@ admission fencing remain separate integration coverage. No new in-memory helper is presented as persistence or crash evidence. The source-close fault wrapper only tests transaction outcome/response ordering in the component engine. -A race-enabled test build was attempted but blocked by the host SDK/linker: -`Security.tbd: unknown architecture arm64e.x1`. This is not a successful race run. -Normal tests use `CGO_ENABLED=0`; rerun the race check on a working C toolchain. +The package tests above, `make property-test` at 5,000 cases, and the four gated +counterexamples also run under the race detector with no data race reported. +On a macOS host whose default SDK is newer than its linker, the race build fails +with `unknown architecture arm64e.x1`; set `SDKROOT` to the SDK matching the +installed Command Line Tools. From fceb1ccb095b78651e55d6a6df45f1a3f0ba86ba Mon Sep 17 00:00:00 2001 From: chaptersix <13949480+chaptersix@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:24:24 -0600 Subject: [PATCH 11/22] Run the scheduler harness on rapid's test handle Add chasmtest.TB, the subset of testing.TB that the test engine, gomock and testlogger need, which both *testing.T and *rapid.T satisfy. The engine, InvariantCheck and the schedulertest driver take it, and the property tests build their driver from the *rapid.T so harness failures are shrunk and recorded in rapid's failfile. Co-Authored-By: Claude Fable 5.2 --- chasm/chasmtest/test_engine.go | 18 ++++++++++++++---- chasm/chasmtest/test_engine_test.go | 6 +++--- chasm/lib/scheduler/generator_wakeup_test.go | 2 +- chasm/lib/scheduler/helper_test.go | 2 +- chasm/lib/scheduler/invariant_test.go | 7 ++++--- chasm/lib/scheduler/schedulertest/driver.go | 9 ++++----- .../schedulertest/driver_smoke_test.go | 6 ++++++ chasm/lib/scheduler/schedulertest/external.go | 4 ++-- .../schedulertest/planner_properties_test.go | 12 ++++++------ .../scheduler/schedulertest/property_test.go | 8 ++++---- 10 files changed, 45 insertions(+), 29 deletions(-) diff --git a/chasm/chasmtest/test_engine.go b/chasm/chasmtest/test_engine.go index 8b33b40c3cf..aaa1d1d61b7 100644 --- a/chasm/chasmtest/test_engine.go +++ b/chasm/chasmtest/test_engine.go @@ -6,7 +6,6 @@ import ( "fmt" "maps" "sync" - "testing" "time" enumspb "go.temporal.io/api/enums/v1" @@ -25,19 +24,30 @@ import ( ) type ( + // TB is the subset of [testing.TB] the engine, gomock controllers and + // testlogger need. Both *testing.T and *rapid.T satisfy it, so an engine can + // be built inside a rapid property and have failures flow through rapid's + // shrinking. It is declared here, rather than imported from rapid, so this + // package does not depend on rapid. + TB interface { + testlogger.CleanupCapableT + Errorf(format string, args ...any) + Context() context.Context + } + EngineOption func(*Engine) // InvariantCheck validates an execution after a successful transition. The // node is clean, so checks may inspect its snapshot without affecting the // state being validated. - InvariantCheck func(*testing.T, *chasm.Node, chasm.RootComponent) + InvariantCheck func(TB, *chasm.Node, chasm.RootComponent) // Engine is a lightweight in memory CHASM engine for unit tests. It implements // [chasm.Engine] and supports the full set of conflict and reuse policies, as // well as blocking [PollComponent] with [NotifyExecution], matching the behavior // of the production engine as closely as possible without persistence or shard logic. Engine struct { - t *testing.T + t TB registry *chasm.Registry logger log.Logger metrics metrics.Handler @@ -107,7 +117,7 @@ var defaultTransitionOptions = chasm.TransitionOptions{ var _ chasm.Engine = (*Engine)(nil) func NewEngine( - t *testing.T, + t TB, registry *chasm.Registry, opts ...EngineOption, ) *Engine { diff --git a/chasm/chasmtest/test_engine_test.go b/chasm/chasmtest/test_engine_test.go index 48dc3c54708..390595d2ef7 100644 --- a/chasm/chasmtest/test_engine_test.go +++ b/chasm/chasmtest/test_engine_test.go @@ -50,7 +50,7 @@ func TestTasksArePhysicallyGenerated(t *testing.T) { func TestInvariantCheckRunsAfterSuccessfulTransitions(t *testing.T) { var calls []string check := func(name string) chasmtest.InvariantCheck { - return func(checkT *testing.T, node *chasm.Node, root chasm.RootComponent) { + return func(checkT chasmtest.TB, node *chasm.Node, root chasm.RootComponent) { checkT.Helper() require.Same(checkT, t, checkT) require.IsType(checkT, &tests.PayloadStore{}, root) @@ -122,7 +122,7 @@ func TestInvariantCheckRunsAfterSuccessfulTransitions(t *testing.T) { // sub-components its update adds must become nodes in the tree, not only exist in memory. func TestUpdateWithStartSyncsSubComponentsAddedByUpdate(t *testing.T) { var nodePaths []string - check := func(_ *testing.T, node *chasm.Node, _ chasm.RootComponent) { + check := func(_ chasmtest.TB, node *chasm.Node, _ chasm.RootComponent) { nodePaths = slices.Collect(maps.Keys(node.Snapshot(nil).Nodes)) } e, _ := startStoreWithEngineOptions(t, 0, []chasmtest.EngineOption{chasmtest.WithInvariantCheck(check)}) @@ -368,7 +368,7 @@ func TestRecordsCallerPrincipal(t *testing.T) { worker := &commonpb.Principal{Type: "jwt", Name: "worker"} var rootAttributes *persistencespb.ChasmComponentAttributes - captureRoot := func(_ *testing.T, node *chasm.Node, _ chasm.RootComponent) { + captureRoot := func(_ chasmtest.TB, node *chasm.Node, _ chasm.RootComponent) { rootAttributes = node.Snapshot(nil).Nodes[""].GetMetadata().GetComponentAttributes() } diff --git a/chasm/lib/scheduler/generator_wakeup_test.go b/chasm/lib/scheduler/generator_wakeup_test.go index c895cbb0a12..b35cbd75f03 100644 --- a/chasm/lib/scheduler/generator_wakeup_test.go +++ b/chasm/lib/scheduler/generator_wakeup_test.go @@ -54,7 +54,7 @@ func TestGeneratorNextWakeupTimeStaysAtPendingJitteredFire(t *testing.T) { require.NoError(t, err) var timerDeadlines []time.Time observeTimers := func(config *engineTestConfig) { - config.engineOpts = append(config.engineOpts, chasmtest.WithInvariantCheck(func(_ *testing.T, node *chasm.Node, _ chasm.RootComponent) { + config.engineOpts = append(config.engineOpts, chasmtest.WithInvariantCheck(func(_ chasmtest.TB, node *chasm.Node, _ chasm.RootComponent) { timerDeadlines = nil generator := node.Snapshot(nil).Nodes[generatorPath] for _, task := range generator.GetMetadata().GetComponentAttributes().GetPureTasks() { diff --git a/chasm/lib/scheduler/helper_test.go b/chasm/lib/scheduler/helper_test.go index b73ed03e28f..295a431d761 100644 --- a/chasm/lib/scheduler/helper_test.go +++ b/chasm/lib/scheduler/helper_test.go @@ -278,7 +278,7 @@ func newTestEngineContextFromConfig( require.NoError(t, registry.Register(newTestLibrary(logger, specProcessor))) config.engineOpts = append(config.engineOpts, chasmtest.WithInvariantCheck( - func(t *testing.T, node *chasm.Node, root chasm.RootComponent) { + func(t chasmtest.TB, node *chasm.Node, root chasm.RootComponent) { requireValidSchedulerState(t, registry, node, root) }, )) diff --git a/chasm/lib/scheduler/invariant_test.go b/chasm/lib/scheduler/invariant_test.go index b35b08be0f2..699eb70cc95 100644 --- a/chasm/lib/scheduler/invariant_test.go +++ b/chasm/lib/scheduler/invariant_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/require" "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/chasmtest" "go.temporal.io/server/chasm/lib/scheduler" "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" "google.golang.org/protobuf/types/known/timestamppb" @@ -139,7 +140,7 @@ func idleTaskCount(registry *chasm.Registry, node *chasm.Node) int { // Only asserted in this direction. Sentinels legitimately arm an idle task // without setting IdleCloseTime (see NewSentinel), so an armed task with no // declared deadline is not a defect. -func requireIdleCloseTimeBacked(t *testing.T, registry *chasm.Registry, node *chasm.Node, sched *scheduler.Scheduler) { +func requireIdleCloseTimeBacked(t chasmtest.TB, registry *chasm.Registry, node *chasm.Node, sched *scheduler.Scheduler) { t.Helper() if sched.GetSentinel() || sched.GetClosed() || sched.GetIdleCloseTime() == nil { @@ -153,7 +154,7 @@ func requireIdleCloseTimeBacked(t *testing.T, registry *chasm.Registry, node *ch } // requireNotStuck asserts the stuckness invariant against a clean tree. -func requireNotStuck(t *testing.T, node *chasm.Node, sched *scheduler.Scheduler) { +func requireNotStuck(t chasmtest.TB, node *chasm.Node, sched *scheduler.Scheduler) { t.Helper() total, byPath := liveTaskCount(node) @@ -162,7 +163,7 @@ func requireNotStuck(t *testing.T, node *chasm.Node, sched *scheduler.Scheduler) } func requireValidSchedulerState( - t *testing.T, + t chasmtest.TB, registry *chasm.Registry, node *chasm.Node, root chasm.RootComponent, diff --git a/chasm/lib/scheduler/schedulertest/driver.go b/chasm/lib/scheduler/schedulertest/driver.go index da405adb7d9..6789493cc65 100644 --- a/chasm/lib/scheduler/schedulertest/driver.go +++ b/chasm/lib/scheduler/schedulertest/driver.go @@ -8,7 +8,6 @@ import ( "maps" "slices" "sync" - "testing" "time" commonpb "go.temporal.io/api/common/v1" @@ -63,7 +62,7 @@ type Driver struct { recorded map[string]ExternalStart initialActionCount int64 - T *testing.T + T chasmtest.TB Engine *chasmtest.Engine Registry *chasm.Registry TimeSource *clock.EventTimeSource @@ -165,7 +164,7 @@ func WithMaxRunDuration(d time.Duration) DriverOption { // NewDriver constructs a Driver and starts a scheduler execution via the // production CreateScheduler path. startTime defaults to a fixed instant for // determinism. -func NewDriver(t *testing.T, opts ...DriverOption) *Driver { +func NewDriver(t chasmtest.TB, opts ...DriverOption) *Driver { t.Helper() d, cfg := newDriverScaffold(t, opts...) @@ -187,7 +186,7 @@ func NewDriver(t *testing.T, opts ...DriverOption) *Driver { // NewDriverFromMigration constructs a Driver and starts a scheduler execution // from a migrated V1 state via the production CreateSchedulerFromMigration path. // This is the V1->V2 import side of a migration round trip. -func NewDriverFromMigration(t *testing.T, req *schedulerpb.CreateFromMigrationStateRequest, opts ...DriverOption) *Driver { +func NewDriverFromMigration(t chasmtest.TB, req *schedulerpb.CreateFromMigrationStateRequest, opts ...DriverOption) *Driver { t.Helper() d, _ := newDriverScaffold(t, opts...) d.startExecution(func(mctx chasm.MutableContext) (*scheduler.Scheduler, error) { @@ -198,7 +197,7 @@ func NewDriverFromMigration(t *testing.T, req *schedulerpb.CreateFromMigrationSt // newDriverScaffold builds the engine, registry, clients, and time source, but // does not start an execution. -func newDriverScaffold(t *testing.T, opts ...DriverOption) (*Driver, *driverConfig) { +func newDriverScaffold(t chasmtest.TB, opts ...DriverOption) (*Driver, *driverConfig) { t.Helper() cfg := &driverConfig{ diff --git a/chasm/lib/scheduler/schedulertest/driver_smoke_test.go b/chasm/lib/scheduler/schedulertest/driver_smoke_test.go index 4f80dfa219a..d3d52d6d002 100644 --- a/chasm/lib/scheduler/schedulertest/driver_smoke_test.go +++ b/chasm/lib/scheduler/schedulertest/driver_smoke_test.go @@ -6,8 +6,10 @@ import ( "github.com/stretchr/testify/require" "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/chasmtest" "go.temporal.io/server/chasm/lib/scheduler" "go.temporal.io/server/chasm/lib/scheduler/schedulertest" + "pgregory.net/rapid" ) // TestDriver_RunsIntervalScheduleForward verifies the driver fires the @@ -70,3 +72,7 @@ func TestDriver_LimitedActionsExhaustedClosesAndQuiesces(t *testing.T) { require.True(t, s.Closed, "schedule should be closed after idle task fires") }) } + +// *rapid.T must stay usable as the driver's test handle: harness failures then +// flow through rapid's shrinking. +var _ chasmtest.TB = (*rapid.T)(nil) diff --git a/chasm/lib/scheduler/schedulertest/external.go b/chasm/lib/scheduler/schedulertest/external.go index 1e3054e0fdd..2988ecd7fef 100644 --- a/chasm/lib/scheduler/schedulertest/external.go +++ b/chasm/lib/scheduler/schedulertest/external.go @@ -5,12 +5,12 @@ import ( "fmt" "maps" "sync" - "testing" commonpb "go.temporal.io/api/common/v1" "go.temporal.io/api/workflowservice/v1" persistencespb "go.temporal.io/server/api/persistence/v1" "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/chasmtest" "go.temporal.io/server/chasm/lib/scheduler" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -50,7 +50,7 @@ func (l *externalLedger) snapshot() map[string]ExternalStart { // test can deliver a callback or lose the response after external acceptance. type StartCall struct { Request *workflowservice.StartWorkflowExecutionRequest - t testing.TB + t chasmtest.TB result chan startResponse ledger *externalLedger } diff --git a/chasm/lib/scheduler/schedulertest/planner_properties_test.go b/chasm/lib/scheduler/schedulertest/planner_properties_test.go index 708687cdef9..3d09f57bd8d 100644 --- a/chasm/lib/scheduler/schedulertest/planner_properties_test.go +++ b/chasm/lib/scheduler/schedulertest/planner_properties_test.go @@ -14,12 +14,12 @@ func TestDeferredBufferOneProperty(t *testing.T) { schedule := schedulertest.DefaultSchedule() schedule.State.Paused = true schedule.Policies.OverlapPolicy = enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ONE - d := schedulertest.NewDriver(t, schedulertest.WithSchedule(schedule)) - require.NoError(t, d.TriggerImmediately()) - require.True(t, d.StepSideEffect()) + d := schedulertest.NewDriver(rt, schedulertest.WithSchedule(schedule)) + require.NoError(rt, d.TriggerImmediately()) + require.True(rt, d.StepSideEffect()) n := rapid.IntRange(1, 20).Draw(rt, "manualOccurrences") for range n { - require.NoError(t, d.TriggerImmediately()) + require.NoError(rt, d.TriggerImmediately()) } snap := d.Snapshot() if len(snap.PendingStarts) != 1 || snap.OverlapSkipped != int64(n-1) { @@ -28,8 +28,8 @@ func TestDeferredBufferOneProperty(t *testing.T) { if snap.ActionCount != 1 { rt.Fatalf("pending work consumed starts: %d", snap.ActionCount) } - require.True(t, d.CompleteWorkflow()) - require.True(t, d.StepSideEffect()) + require.True(rt, d.CompleteWorkflow()) + require.True(rt, d.StepSideEffect()) if cur := d.Snapshot(); cur.ActionCount != 2 || len(cur.PendingStarts) != 0 { rt.Fatalf("completion failed to release exactly one start: count=%d pending=%d", cur.ActionCount, len(cur.PendingStarts)) } diff --git a/chasm/lib/scheduler/schedulertest/property_test.go b/chasm/lib/scheduler/schedulertest/property_test.go index 91406004c01..d561cca590e 100644 --- a/chasm/lib/scheduler/schedulertest/property_test.go +++ b/chasm/lib/scheduler/schedulertest/property_test.go @@ -20,9 +20,9 @@ import ( // failing operation sequence to a minimal reproducer — directly useful for the // "stuck open" bug class. // -// The outer *testing.T owns the engine scaffolding (it requires a real -// testing.TB); invariant violations are reported through the *rapid.T so rapid -// can shrink them. +// The driver is built on the *rapid.T, so harness failures (not just invariant +// violations) are shrunk by rapid and recorded in its failfile, and each +// iteration's mocks and logger are scoped to that iteration. func TestSchedulerProperties(t *testing.T) { rapid.Check(t, func(rt *rapid.T) { start := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) @@ -31,7 +31,7 @@ func TestSchedulerProperties(t *testing.T) { idle := rapid.SampledFrom([]time.Duration{10 * time.Minute, time.Hour, 2 * time.Hour}).Draw(rt, "idleTime") sched := drawSchedule(rt, start, idle) - d := schedulertest.NewDriver(t, + d := schedulertest.NewDriver(rt, schedulertest.WithStartTime(start), schedulertest.WithSchedule(sched), schedulertest.WithConfig(configWith(func(tw *scheduler.Tweakables) { From 2eae3998aa64b04eed0871cd0882c8f2d93173a6 Mon Sep 17 00:00:00 2001 From: chaptersix <13949480+chaptersix@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:29:43 -0600 Subject: [PATCH 12/22] Move the task stepping loop into chasmtest Add chasmtest.TaskRunner, which steps an execution through its own pure and side-effect tasks with a controllable clock, with hooks for component-specific behaviour. The scheduler driver now delegates to it and keeps only scheduler operations, snapshots and invariants. The next pure-task time is read from the tree's pure-task metadata instead of the mock backend, so Engine.NodeForRef, BackendForRef and CloseTransaction are removed. Co-Authored-By: Claude Fable 5.2 --- chasm/chasmtest/task_runner.go | 488 ++++++++++++++++++ chasm/chasmtest/task_runner_test.go | 283 ++++++++++ chasm/chasmtest/test_engine.go | 33 -- chasm/lib/scheduler/schedulertest/driver.go | 314 +++-------- chasm/lib/scheduler/schedulertest/external.go | 40 +- .../scheduler/schedulertest/task_validity.go | 30 +- .../schedulertest/task_validity_test.go | 9 +- 7 files changed, 857 insertions(+), 340 deletions(-) create mode 100644 chasm/chasmtest/task_runner.go create mode 100644 chasm/chasmtest/task_runner_test.go diff --git a/chasm/chasmtest/task_runner.go b/chasm/chasmtest/task_runner.go new file mode 100644 index 00000000000..80fd4c68bf4 --- /dev/null +++ b/chasm/chasmtest/task_runner.go @@ -0,0 +1,488 @@ +package chasmtest + +import ( + "cmp" + "context" + "slices" + "time" + + persistencespb "go.temporal.io/server/api/persistence/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/common/clock" + "go.temporal.io/server/service/history/tasks" + "google.golang.org/protobuf/proto" +) + +// TaskRunner steps one execution on an [Engine] through the tasks it schedules +// for itself, moving a controllable clock forward. It stands in for the queue +// processors that deliver pure and side-effect tasks in production, so a test +// can drive a component through its whole lifecycle deterministically and check +// invariants after every settled transition. +// +// A step fires every pure task due at the current reference time (through +// [Engine.FirePureTasks]) and dispatches every side-effect task whose +// visibility time has arrived, then advances the clock to the next scheduled +// task. A component that keeps re-arming a timer never quiesces, so callers +// bound [TaskRunner.RunToQuiescence] with a step count. +// +// Component-specific behaviour (completing external work before the clock +// passes some deadline, counting RPCs, tolerating particular task errors, +// intercepting the engine a task handler sees) is plugged in through +// [TaskRunnerHooks] and [WithAsyncEngine]. +// +// A TaskRunner is not safe for concurrent use. The exception is the +// Begin/Finish pair, which lets the test goroutine interleave with one task +// running on its own goroutine. +type TaskRunner struct { + t TB + engine *Engine + ref chasm.ComponentRef + clock *clock.EventTimeSource + + maxSideEffectLag time.Duration + hooks TaskRunnerHooks + wrapAsyncEngine func(chasm.Engine) chasm.Engine + + // dispatched holds the physical side-effect tasks already handed out, + // keyed by pointer identity: the engine's backend keeps every physical task + // it ever generated, and executing a task does not remove it. + dispatched map[*tasks.ChasmTask]struct{} + + // inFlight receives the result of the task started by BeginSideEffect. + inFlight chan error +} + +// TaskRunnerHooks are optional callbacks that let a component's test harness +// observe and extend the stepping loop. Any field may be nil. They run on the +// stepping goroutine, in the order the fields are listed for each kind of step. +// +// A side-effect dispatch runs BeforeSideEffect, executes the task, applies +// TolerateSideEffectError, runs AfterSideEffect, commits the transaction (which +// runs the engine's invariant checks), then runs AfterSideEffectCommit and +// AfterStep. A pure-task step fires the due tasks and commits, then runs +// AfterStep. BeforeAdvance runs ahead of every clock move. +type TaskRunnerHooks struct { + // BeforeAdvance runs before the clock moves to the given time and may + // itself move the clock to an earlier instant, for example to complete + // external work that would otherwise run past a deadline. It runs even + // when the clock does not end up moving. + BeforeAdvance func(to time.Time) + // BeforeSideEffect runs after a task has been selected and validated, just + // before it executes. + BeforeSideEffect func() + // AfterSideEffect runs once the task returned, successfully or with a + // tolerated error, before its transaction is committed. + AfterSideEffect func() + // AfterSideEffectCommit runs after a side-effect task's transaction is + // committed, before AfterStep. + AfterSideEffectCommit func() + // AfterStep runs after every committed pure or side-effect step. + AfterStep func() + // TolerateSideEffectError reports whether an error returned by a + // side-effect task is expected and should not fail the test. By default + // every error fails the test. + TolerateSideEffectError func(err error) bool +} + +// TaskRunnerOption configures a [TaskRunner]. +type TaskRunnerOption func(*TaskRunner) + +// WithMaxSideEffectLag bounds how far [TaskRunner.StepPure] may move the clock +// past the oldest undispatched side-effect task, modelling a side-effect queue +// that lags behind the timers. Zero, the default, means unbounded. +func WithMaxSideEffectLag(lag time.Duration) TaskRunnerOption { + return func(r *TaskRunner) { r.maxSideEffectLag = lag } +} + +// WithTaskRunnerHooks installs hooks. See [TaskRunnerHooks]. +func WithTaskRunnerHooks(hooks TaskRunnerHooks) TaskRunnerOption { + return func(r *TaskRunner) { r.hooks = hooks } +} + +// WithAsyncEngine wraps the engine that a task started with +// [TaskRunner.BeginSideEffect] sees, so a test can intercept the calls the +// handler makes back into the engine. Tasks dispatched through the Step methods +// always see the engine itself. +func WithAsyncEngine(wrap func(chasm.Engine) chasm.Engine) TaskRunnerOption { + return func(r *TaskRunner) { r.wrapAsyncEngine = wrap } +} + +// NewTaskRunner returns a runner for the execution identified by ref on e. The +// execution need not exist yet. ts must be the time source the engine was built +// with ([WithTimeSource]), since the runner moves it. +func NewTaskRunner( + t TB, + e *Engine, + ref chasm.ComponentRef, + ts *clock.EventTimeSource, + opts ...TaskRunnerOption, +) *TaskRunner { + r := &TaskRunner{ + t: t, + engine: e, + ref: ref, + clock: ts, + dispatched: make(map[*tasks.ChasmTask]struct{}), + } + for _, opt := range opts { + opt(r) + } + return r +} + +// Step advances the clock to the next scheduled task and fires all tasks due at +// that time: pure tasks first, then every due side-effect task. It returns false +// if nothing was pending, i.e. the execution is quiescent. +func (r *TaskRunner) Step() bool { + r.t.Helper() + + fireTime, ok := r.nextFireTime() + if !ok { + return false + } + r.advanceTo(fireTime) + r.firePureTasks() + for r.StepSideEffect() { + } + return true +} + +// StepPure advances the clock to the next pending pure task and fires every +// pure task due then, leaving side-effect tasks undispatched. This models the +// side-effect queue lagging behind the timers. It returns false if no pure task +// is pending, or if firing it would move the clock more than the configured +// [WithMaxSideEffectLag] past the oldest undispatched side-effect task. +func (r *TaskRunner) StepPure() bool { + r.t.Helper() + + fireTime, ok := r.nextPureFireTime() + if !ok { + return false + } + if oldest, ok := r.oldestSideEffectTime(); ok && r.maxSideEffectLag > 0 && + fireTime.After(oldest.Add(r.maxSideEffectLag)) { + return false + } + r.advanceTo(fireTime) + r.firePureTasks() + return true +} + +// StepSideEffect dispatches the oldest side-effect task that is due at the +// current time, without moving the clock. Tasks that are no longer valid are +// skipped. It returns false if none is due. +func (r *TaskRunner) StepSideEffect() bool { + r.t.Helper() + + next := r.nextSideEffect() + if next == nil { + return false + } + r.beforeSideEffect() + err := r.executeSideEffect(context.Background(), r.engine, next) + r.finishSideEffect(err) + return true +} + +// RunToQuiescence steps until no task is pending or maxSteps is reached. It +// returns the number of steps taken and whether the execution quiesced. +func (r *TaskRunner) RunToQuiescence(maxSteps int) (steps int, quiescent bool) { + r.t.Helper() + for steps = 0; steps < maxSteps; steps++ { + if !r.Step() { + return steps, true + } + } + return steps, false +} + +// HasPendingTask reports whether any task, pure or side-effect, is scheduled +// and still valid. +func (r *TaskRunner) HasPendingTask() bool { + r.t.Helper() + + if _, ok := r.nextPureFireTime(); ok { + return true + } + for _, task := range r.PendingSideEffectTasks() { + if inTree, valid := r.ValidateSideEffectTask(task); inTree && valid { + return true + } + } + return false +} + +// PendingPureTasks returns the pure tasks currently scheduled anywhere in the +// execution's tree, earliest first. The result is a copy. +func (r *TaskRunner) PendingPureTasks() []*persistencespb.ChasmComponentAttributes_Task { + r.t.Helper() + + var out []*persistencespb.ChasmComponentAttributes_Task + for _, node := range r.node().Snapshot(nil).Nodes { + for _, task := range node.GetMetadata().GetComponentAttributes().GetPureTasks() { + out = append(out, proto.Clone(task).(*persistencespb.ChasmComponentAttributes_Task)) + } + } + slices.SortStableFunc(out, func(a, b *persistencespb.ChasmComponentAttributes_Task) int { + return a.GetScheduledTime().AsTime().Compare(b.GetScheduledTime().AsTime()) + }) + return out +} + +// BeginSideEffect selects the next due side-effect task and runs it on its own +// goroutine, through the engine wrapped by [WithAsyncEngine], so the test can +// interleave with its calls back into the engine. It returns false if no task +// is due. The result arrives on [TaskRunner.SideEffectDone]; pass it to +// [TaskRunner.FinishSideEffect]. Only one task may be in flight, and the test +// may touch the engine meanwhile only while the handler is blocked in its +// wrapper. +func (r *TaskRunner) BeginSideEffect() bool { + r.t.Helper() + + if r.inFlight != nil { + r.t.Fatalf("a side-effect task is already in flight") + } + next := r.nextSideEffect() + if next == nil { + return false + } + r.beforeSideEffect() + done := make(chan error, 1) + r.inFlight = done + ctx, cancel := context.WithCancel(r.t.Context()) + r.t.Cleanup(func() { + cancel() + if r.inFlight == done { + <-done + r.inFlight = nil + } + }) + var engine chasm.Engine = r.engine + if r.wrapAsyncEngine != nil { + engine = r.wrapAsyncEngine(engine) + } + go func() { done <- r.executeSideEffect(ctx, engine, next) }() + return true +} + +// SideEffectDone returns the channel on which the task started by +// [TaskRunner.BeginSideEffect] reports its result, or nil if none is in flight. +func (r *TaskRunner) SideEffectDone() <-chan error { + return r.inFlight +} + +// FinishSideEffect completes the in-flight task after its result err was +// received from [TaskRunner.SideEffectDone]: it checks the error, commits the +// transaction and runs the after-step hooks. +func (r *TaskRunner) FinishSideEffect(err error) { + r.t.Helper() + r.inFlight = nil + r.finishSideEffect(err) +} + +// AbandonSideEffect forgets the in-flight task after its result was received +// from [TaskRunner.SideEffectDone] but will not be finished, typically because +// the test is about to fail. +func (r *TaskRunner) AbandonSideEffect() { + r.inFlight = nil +} + +func (r *TaskRunner) node() *chasm.Node { + r.t.Helper() + exec, err := r.engine.executionForRef(r.ref) + if err != nil { + r.t.Fatalf("execution lookup failed: %v", err) + } + return exec.node +} + +// nextPureFireTime returns the earliest scheduled pure task time in the tree. +// +// The tree's own pure-task metadata is the source rather than the engine +// backend's record of the last physical-task deletion: when a transaction +// closes, the node derives that recorded time from exactly this minimum over the +// first pure task of each component, so the two agree whenever the tree is +// clean, which it is between steps. +func (r *TaskRunner) nextPureFireTime() (time.Time, bool) { + r.t.Helper() + var earliest time.Time + have := false + for _, node := range r.node().Snapshot(nil).Nodes { + pure := node.GetMetadata().GetComponentAttributes().GetPureTasks() + if len(pure) == 0 { + continue + } + if at := pure[0].GetScheduledTime().AsTime(); !have || at.Before(earliest) { + earliest, have = at, true + } + } + return earliest, have +} + +// oldestSideEffectTime returns the earliest visibility time among undispatched +// side-effect tasks, if any. +func (r *TaskRunner) oldestSideEffectTime() (time.Time, bool) { + var oldest time.Time + have := false + for _, task := range r.PendingSideEffectTasks() { + if vt := task.GetVisibilityTime(); !have || vt.Before(oldest) { + oldest, have = vt, true + } + } + return oldest, have +} + +// nextFireTime returns the earliest time at which any pending task is due. +func (r *TaskRunner) nextFireTime() (time.Time, bool) { + pure, havePure := r.nextPureFireTime() + side, haveSide := r.oldestSideEffectTime() + switch { + case havePure && haveSide: + if side.Before(pure) { + return side, true + } + return pure, true + case havePure: + return pure, true + default: + return side, haveSide + } +} + +// PendingSideEffectTasks returns the physical side-effect tasks this runner has +// not yet handed out, in the order the production queue would deliver them. A +// task may be pending yet no longer valid; see [TaskRunner.ValidateSideEffectTask]. +func (r *TaskRunner) PendingSideEffectTasks() []*tasks.ChasmTask { + exec, err := r.engine.executionForRef(r.ref) + if err != nil { + r.t.Fatalf("execution lookup failed: %v", err) + } + var out []*tasks.ChasmTask + for category, categoryTasks := range exec.backend.TasksByCategory { + // Visibility tasks are framework-managed (indexing search attributes) and + // processed by a separate queue, not the CHASM side-effect executor; their + // handler panics if invoked directly. Skip them. + if category.ID() == tasks.CategoryVisibility.ID() { + continue + } + for _, t := range categoryTasks { + ct, ok := t.(*tasks.ChasmTask) + if !ok { + continue + } + if _, done := r.dispatched[ct]; done { + continue + } + out = append(out, ct) + } + } + slices.SortStableFunc(out, func(a, b *tasks.ChasmTask) int { + if c := a.GetVisibilityTime().Compare(b.GetVisibilityTime()); c != 0 { + return c + } + if c := cmp.Compare(a.Info.GetTaskVersionedTransition().GetTransitionCount(), b.Info.GetTaskVersionedTransition().GetTransitionCount()); c != 0 { + return c + } + if c := cmp.Compare(a.Info.GetTaskVersionedTransitionOffset(), b.Info.GetTaskVersionedTransitionOffset()); c != 0 { + return c + } + if c := cmp.Compare(a.Info.GetTypeId(), b.Info.GetTypeId()); c != 0 { + return c + } + return slices.Compare(a.Info.GetPath(), b.Info.GetPath()) + }) + return out +} + +// ValidateSideEffectTask reports whether the physical task still corresponds to +// a logical task in the tree (inTree), and whether that logical task's handler +// still considers it valid. It does not mark the task as dispatched. +func (r *TaskRunner) ValidateSideEffectTask(task *tasks.ChasmTask) (inTree, valid bool) { + r.t.Helper() + inTree, valid, err := r.node().ValidateSideEffectTask(context.Background(), task) + if err != nil { + r.t.Fatalf("ValidateSideEffectTask: %v", err) + } + return inTree, valid +} + +// nextSideEffect returns the oldest due, valid side-effect task and marks it +// and every invalid task before it as dispatched. +func (r *TaskRunner) nextSideEffect() *tasks.ChasmTask { + r.t.Helper() + for _, task := range r.PendingSideEffectTasks() { + if task.GetVisibilityTime().After(r.clock.Now()) { + continue + } + r.dispatched[task] = struct{}{} + if inTree, valid := r.ValidateSideEffectTask(task); inTree && valid { + return task + } + } + return nil +} + +func (r *TaskRunner) executeSideEffect(ctx context.Context, engine chasm.Engine, task *tasks.ChasmTask) error { + return r.node().ExecuteSideEffectTask( + chasm.NewEngineContext(ctx, engine), r.ref.ExecutionKey, task, + func(chasm.NodeBackend, chasm.Context, chasm.Component) error { return nil }, + ) +} + +func (r *TaskRunner) beforeSideEffect() { + if r.hooks.BeforeSideEffect != nil { + r.hooks.BeforeSideEffect() + } +} + +func (r *TaskRunner) finishSideEffect(err error) { + r.t.Helper() + if err != nil && (r.hooks.TolerateSideEffectError == nil || !r.hooks.TolerateSideEffectError(err)) { + r.t.Fatalf("ExecuteSideEffectTask failed: %v", err) + } + if r.hooks.AfterSideEffect != nil { + r.hooks.AfterSideEffect() + } + r.commit() + if r.hooks.AfterSideEffectCommit != nil { + r.hooks.AfterSideEffectCommit() + } + r.afterStep() +} + +// firePureTasks fires every pure task due at the current time and commits. +func (r *TaskRunner) firePureTasks() { + r.t.Helper() + if _, err := r.engine.FirePureTasks(r.ref, r.clock.Now()); err != nil { + r.t.Fatalf("FirePureTasks failed: %v", err) + } + r.afterStep() +} + +func (r *TaskRunner) commit() { + r.t.Helper() + exec, err := r.engine.executionForRef(r.ref) + if err == nil { + err = r.engine.closeTransaction(context.Background(), exec) + } + if err != nil { + r.t.Fatalf("closing transaction: %v", err) + } +} + +func (r *TaskRunner) afterStep() { + if r.hooks.AfterStep != nil { + r.hooks.AfterStep() + } +} + +// advanceTo moves the clock to t, never backward, after giving BeforeAdvance a +// chance to act on the way. +func (r *TaskRunner) advanceTo(t time.Time) { + if r.hooks.BeforeAdvance != nil { + r.hooks.BeforeAdvance(t) + } + if t.After(r.clock.Now()) { + r.clock.Update(t) + } +} diff --git a/chasm/chasmtest/task_runner_test.go b/chasm/chasmtest/task_runner_test.go new file mode 100644 index 00000000000..b02199d46f2 --- /dev/null +++ b/chasm/chasmtest/task_runner_test.go @@ -0,0 +1,283 @@ +package chasmtest_test + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.temporal.io/api/serviceerror" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/chasmtest" + "go.temporal.io/server/chasm/lib/tests" + "go.temporal.io/server/chasm/lib/tests/gen/testspb/v1" + "go.temporal.io/server/common/clock" + "go.temporal.io/server/common/log" + "google.golang.org/protobuf/types/known/timestamppb" +) + +var runnerStart = time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) + +type runnerFixture struct { + engine *chasmtest.Engine + ref chasm.ComponentRef + ts *clock.EventTimeSource +} + +// newRunnerFixture starts a payload store holding payload "ttl", whose pure +// task removes it after ttl. +func newRunnerFixture(t *testing.T, ttl time.Duration) *runnerFixture { + t.Helper() + registry := chasm.NewRegistry(log.NewNoopLogger()) + require.NoError(t, registry.Register(&chasm.CoreLibrary{})) + require.NoError(t, registry.Register(tests.Library)) + ts := clock.NewEventTimeSource() + ts.Update(runnerStart) + engine := chasmtest.NewEngine(t, registry, chasmtest.WithTimeSource(ts)) + + key := chasm.ExecutionKey{NamespaceID: "test-ns", BusinessID: "store"} + _, err := chasm.StartExecution(engineContext(engine), key, + func(mc chasm.MutableContext, _ any) (*tests.PayloadStore, error) { + store, err := tests.NewPayloadStore(mc) + if err != nil { + return nil, err + } + return store, addPayload(store, mc, "ttl", ttl) + }, nil) + require.NoError(t, err) + return &runnerFixture{engine: engine, ref: chasm.NewComponentRef[*tests.PayloadStore](key), ts: ts} +} + +// addSideEffect schedules a side-effect task for key at the given offset from +// the start. When present is false the key has no payload, so the task is valid +// but its execution fails with NotFound. +func (f *runnerFixture) addSideEffect(t *testing.T, key string, after time.Duration, present bool) { + t.Helper() + _, err := f.engine.UpdateComponent(engineContext(f.engine), f.ref, + func(mc chasm.MutableContext, c chasm.Component) error { + store := c.(*tests.PayloadStore) + if present { + if err := addPayload(store, mc, key, 0); err != nil { + return err + } + } + at := runnerStart.Add(after) + if store.State.ExpirationTimes == nil { + store.State.ExpirationTimes = make(map[string]*timestamppb.Timestamp) + } + store.State.ExpirationTimes[key] = timestamppb.New(at) + mc.AddTask(store, chasm.TaskAttributes{ScheduledTime: at}, &testspb.TestPayloadTTLSideEffectTask{PayloadKey: key}) + return nil + }) + require.NoError(t, err) +} + +func (f *runnerFixture) hasPayload(t *testing.T, key string) bool { + t.Helper() + var ok bool + require.NoError(t, f.engine.ReadComponent(engineContext(f.engine), f.ref, + func(_ chasm.Context, c chasm.Component) error { + _, ok = c.(*tests.PayloadStore).Payloads[key] + return nil + })) + return ok +} + +func TestTaskRunnerStepAdvancesClockAndFiresPureTasks(t *testing.T) { + f := newRunnerFixture(t, time.Hour) + r := chasmtest.NewTaskRunner(t, f.engine, f.ref, f.ts) + + require.True(t, r.HasPendingTask()) + require.True(t, f.hasPayload(t, "ttl")) + + require.True(t, r.Step()) + require.Equal(t, runnerStart.Add(time.Hour), f.ts.Now()) + require.False(t, f.hasPayload(t, "ttl")) + + require.False(t, r.HasPendingTask()) + require.False(t, r.Step()) + require.Equal(t, runnerStart.Add(time.Hour), f.ts.Now(), "a quiescent step leaves the clock alone") +} + +func TestTaskRunnerRunToQuiescence(t *testing.T) { + f := newRunnerFixture(t, time.Hour) + f.addSideEffect(t, "effect", 2*time.Hour, true) + r := chasmtest.NewTaskRunner(t, f.engine, f.ref, f.ts) + + steps, quiescent := r.RunToQuiescence(10) + require.True(t, quiescent) + require.Equal(t, 2, steps) + require.False(t, f.hasPayload(t, "effect")) + + f = newRunnerFixture(t, time.Hour) + f.addSideEffect(t, "effect", 2*time.Hour, true) + r = chasmtest.NewTaskRunner(t, f.engine, f.ref, f.ts) + steps, quiescent = r.RunToQuiescence(1) + require.False(t, quiescent) + require.Equal(t, 1, steps) +} + +func TestTaskRunnerPendingPureTasksAreEarliestFirst(t *testing.T) { + f := newRunnerFixture(t, time.Hour) + _, err := f.engine.UpdateComponent(engineContext(f.engine), f.ref, + func(mc chasm.MutableContext, c chasm.Component) error { + return addPayload(c.(*tests.PayloadStore), mc, "sooner", time.Minute) + }) + require.NoError(t, err) + r := chasmtest.NewTaskRunner(t, f.engine, f.ref, f.ts) + + var scheduled []time.Time + for _, task := range r.PendingPureTasks() { + scheduled = append(scheduled, task.GetScheduledTime().AsTime()) + } + require.Equal(t, []time.Time{runnerStart.Add(time.Minute), runnerStart.Add(time.Hour)}, scheduled) +} + +func TestTaskRunnerStepPureLeavesSideEffectsUndispatched(t *testing.T) { + for _, tc := range []struct { + name string + lag time.Duration + wantStepPure bool + }{ + {name: "unbounded lag", lag: 0, wantStepPure: true}, + {name: "lag shorter than the gap", lag: 5 * time.Minute, wantStepPure: false}, + {name: "lag longer than the gap", lag: 2 * time.Hour, wantStepPure: true}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newRunnerFixture(t, time.Hour) + f.addSideEffect(t, "effect", 10*time.Minute, true) + r := chasmtest.NewTaskRunner(t, f.engine, f.ref, f.ts, chasmtest.WithMaxSideEffectLag(tc.lag)) + + require.Equal(t, tc.wantStepPure, r.StepPure()) + if tc.wantStepPure { + require.Equal(t, runnerStart.Add(time.Hour), f.ts.Now()) + require.False(t, f.hasPayload(t, "ttl")) + } else { + require.Equal(t, runnerStart, f.ts.Now()) + } + require.True(t, f.hasPayload(t, "effect"), "StepPure never dispatches side-effect tasks") + }) + } +} + +func TestTaskRunnerStepSideEffectDispatchesOnlyDueTasks(t *testing.T) { + f := newRunnerFixture(t, 0) + f.addSideEffect(t, "effect", 10*time.Minute, true) + r := chasmtest.NewTaskRunner(t, f.engine, f.ref, f.ts) + + require.False(t, r.StepSideEffect(), "not due yet") + require.True(t, f.hasPayload(t, "effect")) + + f.ts.Update(runnerStart.Add(10 * time.Minute)) + require.True(t, r.StepSideEffect()) + require.False(t, f.hasPayload(t, "effect")) + require.False(t, r.StepSideEffect(), "a dispatched task is not dispatched again") +} + +func TestTaskRunnerHookOrder(t *testing.T) { + f := newRunnerFixture(t, time.Hour) + f.addSideEffect(t, "effect", time.Hour, true) + + var events []string + record := func(name string) func() { return func() { events = append(events, name) } } + var advanceTo, clockAtAdvance time.Time + r := chasmtest.NewTaskRunner(t, f.engine, f.ref, f.ts, chasmtest.WithTaskRunnerHooks(chasmtest.TaskRunnerHooks{ + BeforeAdvance: func(to time.Time) { + advanceTo, clockAtAdvance = to, f.ts.Now() + events = append(events, "beforeAdvance") + }, + BeforeSideEffect: record("beforeSideEffect"), + AfterSideEffect: record("afterSideEffect"), + AfterSideEffectCommit: record("afterSideEffectCommit"), + AfterStep: record("afterStep"), + })) + + require.True(t, r.Step()) + require.Equal(t, runnerStart.Add(time.Hour), advanceTo) + require.Equal(t, runnerStart, clockAtAdvance, "BeforeAdvance runs before the clock moves") + require.Equal(t, []string{ + "beforeAdvance", + "afterStep", // pure tasks + "beforeSideEffect", "afterSideEffect", "afterSideEffectCommit", "afterStep", + }, events) +} + +func TestTaskRunnerBeforeAdvanceMayMoveClock(t *testing.T) { + f := newRunnerFixture(t, time.Hour) + var removedBeforeMove bool + r := chasmtest.NewTaskRunner(t, f.engine, f.ref, f.ts, chasmtest.WithTaskRunnerHooks(chasmtest.TaskRunnerHooks{ + BeforeAdvance: func(to time.Time) { + f.ts.Update(to.Add(-30 * time.Minute)) + removedBeforeMove = !f.hasPayload(t, "ttl") + }, + })) + require.True(t, r.Step()) + require.False(t, removedBeforeMove) + require.Equal(t, runnerStart.Add(time.Hour), f.ts.Now(), "the runner still lands on the task's time") +} + +func TestTaskRunnerToleratedSideEffectError(t *testing.T) { + f := newRunnerFixture(t, 0) + f.addSideEffect(t, "ghost", time.Minute, false) + f.ts.Update(runnerStart.Add(time.Minute)) + + var tolerated error + var afterSteps int + r := chasmtest.NewTaskRunner(t, f.engine, f.ref, f.ts, chasmtest.WithTaskRunnerHooks(chasmtest.TaskRunnerHooks{ + TolerateSideEffectError: func(err error) bool { + tolerated = err + _, ok := errors.AsType[*serviceerror.NotFound](err) + return ok + }, + AfterStep: func() { afterSteps++ }, + })) + + require.True(t, r.StepSideEffect()) + require.Error(t, tolerated) + require.Equal(t, 1, afterSteps, "a tolerated error still settles the step") +} + +func TestTaskRunnerBeginAndFinishSideEffect(t *testing.T) { + f := newRunnerFixture(t, 0) + f.addSideEffect(t, "effect", time.Minute, true) + + var wrapped, afterSteps int + r := chasmtest.NewTaskRunner(t, f.engine, f.ref, f.ts, + chasmtest.WithAsyncEngine(func(e chasm.Engine) chasm.Engine { + return &countingEngine{Engine: e, updates: &wrapped} + }), + chasmtest.WithTaskRunnerHooks(chasmtest.TaskRunnerHooks{AfterStep: func() { afterSteps++ }}), + ) + + require.Nil(t, r.SideEffectDone()) + require.False(t, r.BeginSideEffect(), "nothing is due yet") + + f.ts.Update(runnerStart.Add(time.Minute)) + require.True(t, r.BeginSideEffect()) + err := <-r.SideEffectDone() + require.NoError(t, err) + require.Equal(t, 0, afterSteps, "the step settles only when finished") + r.FinishSideEffect(err) + + require.Nil(t, r.SideEffectDone()) + require.Equal(t, 1, afterSteps) + require.Equal(t, 1, wrapped, "the handler saw the wrapped engine") + require.False(t, f.hasPayload(t, "effect")) + require.False(t, r.BeginSideEffect()) +} + +type countingEngine struct { + chasm.Engine + updates *int +} + +func (e *countingEngine) UpdateComponent( + ctx context.Context, + ref chasm.ComponentRef, + update func(chasm.MutableContext, chasm.Component) error, + opts ...chasm.TransitionOption, +) ([]byte, error) { + *e.updates++ + return e.Engine.UpdateComponent(ctx, ref, update, opts...) +} diff --git a/chasm/chasmtest/test_engine.go b/chasm/chasmtest/test_engine.go index aaa1d1d61b7..fd836f85de5 100644 --- a/chasm/chasmtest/test_engine.go +++ b/chasm/chasmtest/test_engine.go @@ -701,39 +701,6 @@ func (x *execution) recordRequestID(requestID string) { x.requestIDs[requestID] = struct{}{} } -// NodeForRef returns the CHASM tree node backing the execution identified by ref. -// It is intended for test drivers that dispatch tasks at the node level -// ([chasm.Node.ExecuteSideEffectTask]), which the [Engine] API does not expose. -func (e *Engine) NodeForRef(ref chasm.ComponentRef) (*chasm.Node, error) { - exec, err := e.executionForRef(ref) - if err != nil { - return nil, err - } - return exec.node, nil -} - -// BackendForRef returns the [chasm.MockNodeBackend] backing the execution -// identified by ref, e.g. to read the earliest pending pure task time via -// [chasm.MockNodeBackend.LastDeletePureTaskCall]. -func (e *Engine) BackendForRef(ref chasm.ComponentRef) (*chasm.MockNodeBackend, error) { - exec, err := e.executionForRef(ref) - if err != nil { - return nil, err - } - return exec.backend, nil -} - -// CloseTransaction closes any open transaction on the execution identified by -// ref, commits its transition count, and runs registered invariant checks. -// Pair it with node-level dispatch done through [Engine.NodeForRef]. -func (e *Engine) CloseTransaction(ctx context.Context, ref chasm.ComponentRef) error { - exec, err := e.executionForRef(ref) - if err != nil { - return err - } - return e.closeTransaction(ctx, exec) -} - // executionForRef looks up an execution by the ref's RunID when present, or falls back // to the current run for the business ID when RunID is empty. func (e *Engine) executionForRef(ref chasm.ComponentRef) (*execution, error) { diff --git a/chasm/lib/scheduler/schedulertest/driver.go b/chasm/lib/scheduler/schedulertest/driver.go index 6789493cc65..7ed462e163c 100644 --- a/chasm/lib/scheduler/schedulertest/driver.go +++ b/chasm/lib/scheduler/schedulertest/driver.go @@ -1,12 +1,10 @@ package schedulertest import ( - "cmp" "context" "errors" "fmt" "maps" - "slices" "sync" "time" @@ -31,33 +29,25 @@ import ( "go.temporal.io/server/common/metrics/metricstest" "go.temporal.io/server/common/testing/mockapi/workflowservicemock/v1" "go.temporal.io/server/common/testing/testlogger" - "go.temporal.io/server/service/history/tasks" "go.uber.org/mock/gomock" "google.golang.org/grpc" "google.golang.org/protobuf/types/known/timestamppb" ) -// maxKeyFireTime is the sentinel returned by MockNodeBackend.LastDeletePureTaskCall -// when the tree has no pending pure task. It matches the value passed by -// Node.closeTransactionGeneratePhysicalPureTask in that case. -var maxKeyFireTime = tasks.MaximumKey.FireTime - // Driver builds a real CHASM scheduler on top of a [chasmtest.Engine] and steps // it forward through every task it schedules for itself, advancing a controllable // clock. After each settled transition it invokes AfterStep (if set), which is // where invariant assertions are plugged in. // -// A "step" fires every pure task due at the current reference time (via -// [chasm.Node.EachPureTask]) and every side-effect task whose visibility time has -// arrived (via [chasm.Node.ExecuteSideEffectTask]), then advances the clock to the -// next scheduled task. A healthy interval schedule never truly quiesces (it always +// A "step", run by a [chasmtest.TaskRunner], fires every pure task due at the +// current reference time and every side-effect task whose visibility time has +// arrived, then advances the clock to the next scheduled task. A healthy interval schedule never truly quiesces (it always // re-arms a generator task), so callers bound the run with maxSteps; quiescence is // reached only for paused/exhausted/closed schedules. type Driver struct { recordCalls chan *RecordCall external *externalLedger startCalls chan *StartCall - sideEffectDone chan error actionsBefore int recorded map[string]ExternalStart initialActionCount int64 @@ -79,9 +69,8 @@ type Driver struct { ref chasm.ComponentRef execKey chasm.ExecutionKey - // dispatchedSideEffects tracks physical side-effect tasks already executed, - // keyed by pointer identity, since MockNodeBackend accumulates tasks append-only. - dispatchedSideEffects map[*tasks.ChasmTask]struct{} + // runner steps the execution through its pure and side-effect tasks. + runner *chasmtest.TaskRunner // actionRPCs points at the running count of action RPCs (StartWorkflowExecution // + TerminateWorkflowExecution + RequestCancelWorkflowExecution) made through @@ -100,10 +89,6 @@ type Driver struct { // starts from concurrent goroutines. rpcMu *sync.Mutex - // maxSideEffectLag caps how far StepPure may move the clock past the oldest - // undispatched side-effect task. Zero means unbounded. - maxSideEffectLag time.Duration - // maxRunDuration auto-completes a started workflow before the clock moves // past its StartTime plus this. Zero means workflows only complete through // CompleteWorkflow. @@ -301,23 +286,24 @@ func newDriverScaffold(t chasmtest.TB, opts ...DriverOption) (*Driver, *driverCo d := &Driver{ recordCalls: make(chan *RecordCall), external: external, startCalls: startCalls, recorded: make(map[string]ExternalStart), - T: t, - Engine: engine, - Registry: registry, - TimeSource: ts, - Logger: logger, - Frontend: frontend, - History: history, - Config: cfg.config, - dispatchedSideEffects: make(map[*tasks.ChasmTask]struct{}), - actionRPCs: actionRPCs, - startRequests: startRequests, - rpcMu: rpcMu, - metricsCapture: capture, - maxSideEffectLag: cfg.maxLag, - maxRunDuration: cfg.maxRun, - AfterStep: cfg.afterStep, + T: t, + Engine: engine, + Registry: registry, + TimeSource: ts, + Logger: logger, + Frontend: frontend, + History: history, + Config: cfg.config, + actionRPCs: actionRPCs, + startRequests: startRequests, + rpcMu: rpcMu, + metricsCapture: capture, + maxRunDuration: cfg.maxRun, + AfterStep: cfg.afterStep, + execKey: chasm.ExecutionKey{NamespaceID: NamespaceID, BusinessID: ScheduleID}, } + d.ref = chasm.NewComponentRef[*scheduler.Scheduler](d.execKey) + d.runner = d.newTaskRunner(cfg.maxLag) return d, cfg } @@ -325,7 +311,6 @@ func newDriverScaffold(t chasmtest.TB, opts ...DriverOption) (*Driver, *driverCo func (d *Driver) startExecution(startFn func(chasm.MutableContext) (*scheduler.Scheduler, error)) { d.T.Helper() - d.execKey = chasm.ExecutionKey{NamespaceID: NamespaceID, BusinessID: ScheduleID} engineCtx := chasm.NewEngineContext(context.Background(), d.Engine) _, err := chasm.StartExecution(engineCtx, d.execKey, func(mctx chasm.MutableContext, _ *struct{}) (*scheduler.Scheduler, error) { @@ -335,7 +320,6 @@ func (d *Driver) startExecution(startFn func(chasm.MutableContext) (*scheduler.S d.T.Fatalf("failed to start scheduler execution: %v", err) } - d.ref = chasm.NewComponentRef[*scheduler.Scheduler](d.execKey) d.ReadScheduler(func(s *scheduler.Scheduler, _ chasm.Context) { d.initialActionCount = s.Info.GetActionCount() }) } @@ -361,204 +345,62 @@ func (d *Driver) ReadScheduler(fn func(s *scheduler.Scheduler, ctx chasm.Context } } -// node returns the CHASM tree node backing the scheduler execution. -func (d *Driver) node() *chasm.Node { - d.T.Helper() - node, err := d.Engine.NodeForRef(d.ref) - if err != nil { - d.T.Fatalf("NodeForRef failed: %v", err) - } - return node -} - -func (d *Driver) backend() *chasm.MockNodeBackend { - d.T.Helper() - backend, err := d.Engine.BackendForRef(d.ref) - if err != nil { - d.T.Fatalf("BackendForRef failed: %v", err) - } - return backend -} - -// nextPureFireTime returns the earliest pending pure task time, if any. It is -// recorded by closeTransactionGeneratePhysicalPureTask via -// DeleteCHASMPureTasks(earliestPureTaskTime), or MaximumKey.FireTime if none. -func (d *Driver) nextPureFireTime() (time.Time, bool) { - pureTime := d.backend().LastDeletePureTaskCall() - if pureTime.IsZero() || pureTime.Equal(maxKeyFireTime) { - return time.Time{}, false - } - return pureTime, true -} - -// oldestSideEffectTime returns the earliest visibility time among undispatched -// side-effect tasks, if any. -func (d *Driver) oldestSideEffectTime() (time.Time, bool) { - var oldest time.Time - have := false - for _, t := range d.pendingSideEffectTasks() { - if vt := t.GetVisibilityTime(); !have || vt.Before(oldest) { - oldest, have = vt, true - } - } - return oldest, have -} - -// nextFireTime returns the earliest time at which any pending task (pure or -// side-effect) is due, and whether any task is pending at all. -func (d *Driver) nextFireTime() (time.Time, bool) { - pure, havePure := d.nextPureFireTime() - side, haveSide := d.oldestSideEffectTime() - switch { - case havePure && haveSide: - if side.Before(pure) { - return side, true - } - return pure, true - case havePure: - return pure, true - default: - return side, haveSide - } -} - -// pendingSideEffectTasks returns physical side-effect tasks that have not yet -// been dispatched by this driver. -func (d *Driver) pendingSideEffectTasks() []*tasks.ChasmTask { - backend := d.backend() - var out []*tasks.ChasmTask - for category, categoryTasks := range backend.TasksByCategory { - // Visibility tasks are framework-managed (indexing search attributes) and - // processed by a separate queue, not the CHASM side-effect executor; their - // handler panics if invoked directly. Skip them. - if category.ID() == tasks.CategoryVisibility.ID() { - continue - } - for _, t := range categoryTasks { - ct, ok := t.(*tasks.ChasmTask) - if !ok { - continue - } - if _, done := d.dispatchedSideEffects[ct]; done { - continue - } - out = append(out, ct) - } - } - slices.SortStableFunc(out, func(a, b *tasks.ChasmTask) int { - if cmp := a.GetVisibilityTime().Compare(b.GetVisibilityTime()); cmp != 0 { - return cmp - } - if c := cmp.Compare(a.Info.GetTaskVersionedTransition().GetTransitionCount(), b.Info.GetTaskVersionedTransition().GetTransitionCount()); c != 0 { - return c - } - if c := cmp.Compare(a.Info.GetTaskVersionedTransitionOffset(), b.Info.GetTaskVersionedTransitionOffset()); c != 0 { - return c - } - if c := cmp.Compare(a.Info.GetTypeId(), b.Info.GetTypeId()); c != 0 { - return c - } - return slices.Compare(a.Info.GetPath(), b.Info.GetPath()) - }) - return out -} - // Step advances the clock to the next scheduled task and fires all tasks due at // that time: pure tasks first, then every due side-effect task. It returns false // if nothing was pending (the schedule is quiescent). func (d *Driver) Step() bool { d.T.Helper() - - fireTime, have := d.nextFireTime() - if !have { - return false - } - d.advanceTo(fireTime) - d.firePureTasks() - for d.StepSideEffect() { - } - return true + return d.runner.Step() } // StepPure advances the clock to the next pending pure task and fires every pure // task due then, leaving side-effect tasks undispatched. This models // InvokerExecuteTask lagging behind the scheduler's timers. It returns false if // no pure task is pending, or if firing it would move the clock more than -// maxSideEffectLag past the oldest undispatched side-effect task. +// the configured max side-effect lag past the oldest undispatched side-effect task. func (d *Driver) StepPure() bool { d.T.Helper() - - fireTime, have := d.nextPureFireTime() - if !have { - return false - } - if oldest, ok := d.oldestSideEffectTime(); ok && d.maxSideEffectLag > 0 && - fireTime.After(oldest.Add(d.maxSideEffectLag)) { - return false - } - d.advanceTo(fireTime) - d.firePureTasks() - return true + return d.runner.StepPure() } // StepSideEffect dispatches the oldest side-effect task that is due at the // current time, without moving the clock. It returns false if none is due. func (d *Driver) StepSideEffect() bool { d.T.Helper() - next := d.nextSideEffect() - if next == nil { - return false - } - before := d.actionCount() - err := d.executeSideEffect(next) - d.finishSideEffect(before, err) - return true -} - -func (d *Driver) nextSideEffect() *tasks.ChasmTask { - for _, task := range d.pendingSideEffectTasks() { - if task.GetVisibilityTime().After(d.Now()) { - continue - } - d.dispatchedSideEffects[task] = struct{}{} - inTree, valid, err := d.node().ValidateSideEffectTask(context.Background(), task) - if err != nil { - d.T.Fatalf("ValidateSideEffectTask: %v", err) - } - if inTree && valid { - return task - } - } - return nil -} - -func (d *Driver) executeSideEffect(task *tasks.ChasmTask) error { - return d.node().ExecuteSideEffectTask( - chasm.NewEngineContext(context.Background(), d.Engine), d.execKey, task, - func(chasm.NodeBackend, chasm.Context, chasm.Component) error { return nil }, + return d.runner.StepSideEffect() +} + +// newTaskRunner wires the scheduler-specific behaviour into a generic task +// runner. +func (d *Driver) newTaskRunner(maxLag time.Duration) *chasmtest.TaskRunner { + return chasmtest.NewTaskRunner(d.T, d.Engine, d.ref, d.TimeSource, + chasmtest.WithMaxSideEffectLag(maxLag), + chasmtest.WithAsyncEngine(func(engine chasm.Engine) chasm.Engine { + return &recordingEngine{Engine: engine, calls: d.recordCalls} + }), + chasmtest.WithTaskRunnerHooks(chasmtest.TaskRunnerHooks{ + BeforeAdvance: d.completeWorkflowsBefore, + BeforeSideEffect: func() { d.actionsBefore = d.actionCount() }, + // A side-effect task may find the schedule already closed and gone. + TolerateSideEffectError: func(err error) bool { + _, ok := errors.AsType[*serviceerror.NotFound](err) + return ok + }, + AfterSideEffect: func() { d.maxActionsObserved = max(d.maxActionsObserved, d.actionCount()-d.actionsBefore) }, + AfterSideEffectCommit: d.observeRecordedStarts, + AfterStep: func() { + if d.AfterStep != nil { + d.AfterStep(d) + } + }, + }), ) } -func (d *Driver) finishSideEffect(before int, err error) { - d.T.Helper() - if err != nil { - if _, ok := errors.AsType[*serviceerror.NotFound](err); !ok { - d.T.Fatalf("ExecuteSideEffectTask failed: %v", err) - } - } - d.maxActionsObserved = max(d.maxActionsObserved, d.actionCount()-before) - if err := d.Engine.CloseTransaction(context.Background(), d.ref); err != nil { - d.T.Fatalf("CloseTransaction: %v", err) - } - d.observeRecordedStarts() - if d.AfterStep != nil { - d.AfterStep(d) - } -} - -// advanceTo moves the clock to t, never backward, first completing any -// workflow that would exceed maxRunDuration on the way. -func (d *Driver) advanceTo(t time.Time) { +// completeWorkflowsBefore completes any workflow that would exceed +// maxRunDuration before the clock moves to t, moving the clock to the deadline +// first. +func (d *Driver) completeWorkflowsBefore(t time.Time) { for d.maxRunDuration > 0 { startTime, ok := d.oldestRunningStartTime() if !ok { @@ -576,9 +418,6 @@ func (d *Driver) advanceTo(t time.Time) { for d.StepSideEffect() { } } - if t.After(d.Now()) { - d.TimeSource.Update(t) - } } // oldestRunningStartTime returns the StartTime of the oldest started workflow @@ -645,55 +484,20 @@ func (d *Driver) CompleteWorkflow() bool { return completed } -// firePureTasks fires every pure task due at the current time and commits. -func (d *Driver) firePureTasks() { - d.T.Helper() - now := d.Now() - pureCtx := chasm.NewEngineContext(context.Background(), d.Engine) - err := d.node().EachPureTask(now, func(executor chasm.NodePureTask, attrs chasm.TaskAttributes, taskInstance any) (bool, error) { - return executor.ExecutePureTask(pureCtx, attrs, taskInstance) - }) - if err != nil { - d.T.Fatalf("EachPureTask failed: %v", err) - } - if err := d.Engine.CloseTransaction(context.Background(), d.ref); err != nil { - d.T.Fatalf("CloseTransaction after pure tasks failed: %v", err) - } - if d.AfterStep != nil { - d.AfterStep(d) - } -} - // RunToQuiescence steps until no task is pending or maxSteps is reached. It // returns the number of steps taken and whether the schedule reached quiescence // (no pending task). A normal running schedule never quiesces and will stop at // maxSteps with quiescent == false. func (d *Driver) RunToQuiescence(maxSteps int) (steps int, quiescent bool) { d.T.Helper() - for steps = 0; steps < maxSteps; steps++ { - if !d.Step() { - return steps, true - } - } - return steps, false + return d.runner.RunToQuiescence(maxSteps) } // HasPendingTask reports whether any task (pure or side-effect) is currently // scheduled to fire. func (d *Driver) HasPendingTask() bool { - if _, have := d.nextPureFireTime(); have { - return true - } - for _, task := range d.pendingSideEffectTasks() { - inTree, valid, err := d.node().ValidateSideEffectTask(context.Background(), task) - if err != nil { - d.T.Fatalf("ValidateSideEffectTask: %v", err) - } - if inTree && valid { - return true - } - } - return false + d.T.Helper() + return d.runner.HasPendingTask() } // Patch applies a SchedulePatch through the production Scheduler.Patch path in a diff --git a/chasm/lib/scheduler/schedulertest/external.go b/chasm/lib/scheduler/schedulertest/external.go index 2988ecd7fef..d84907ab65c 100644 --- a/chasm/lib/scheduler/schedulertest/external.go +++ b/chasm/lib/scheduler/schedulertest/external.go @@ -81,31 +81,10 @@ func WithControlledStarts() DriverOption { // handler is waiting at NextRecord. func (d *Driver) BeginSideEffect() bool { d.T.Helper() - if d.startCalls == nil || d.sideEffectDone != nil { - d.T.Fatal("controlled starts required; only one task may be in flight") + if d.startCalls == nil { + d.T.Fatal("controlled starts required") } - next := d.nextSideEffect() - if next == nil { - return false - } - d.actionsBefore = d.actionCount() - done := make(chan error, 1) - d.sideEffectDone = done - ctx, cancel := context.WithCancel(d.T.Context()) - d.T.Cleanup(func() { - cancel() - if d.sideEffectDone == done { - <-done - d.sideEffectDone = nil - } - }) - node := d.node() - go func() { - engine := &recordingEngine{Engine: d.Engine, calls: d.recordCalls} - done <- node.ExecuteSideEffectTask(chasm.NewEngineContext(ctx, engine), d.execKey, next, - func(chasm.NodeBackend, chasm.Context, chasm.Component) error { return nil }) - }() - return true + return d.runner.BeginSideEffect() } func (d *Driver) NextStart() *StartCall { @@ -113,8 +92,8 @@ func (d *Driver) NextStart() *StartCall { select { case call := <-d.startCalls: return call - case err := <-d.sideEffectDone: - d.sideEffectDone = nil + case err := <-d.runner.SideEffectDone(): + d.runner.AbandonSideEffect() d.T.Fatalf("task finished before expected start: %v", err) case <-d.T.Context().Done(): d.T.Fatal("waiting for start: test canceled") @@ -132,9 +111,8 @@ func (d *Driver) FinishSideEffect() { } case call := <-d.startCalls: d.T.Fatalf("unconsumed start RPC for request %s; call NextStart before FinishSideEffect", call.Request.GetRequestId()) - case err := <-d.sideEffectDone: - d.sideEffectDone = nil - d.finishSideEffect(d.actionsBefore, err) + case err := <-d.runner.SideEffectDone(): + d.runner.FinishSideEffect(err) return case <-d.T.Context().Done(): d.T.Fatal("waiting for side effect: test canceled") @@ -149,8 +127,8 @@ func (d *Driver) NextRecord() *RecordCall { select { case call := <-d.recordCalls: return call - case err := <-d.sideEffectDone: - d.sideEffectDone = nil + case err := <-d.runner.SideEffectDone(): + d.runner.AbandonSideEffect() d.T.Fatalf("task finished before result recording: %v", err) case <-d.T.Context().Done(): d.T.Fatal("waiting for recording: test canceled") diff --git a/chasm/lib/scheduler/schedulertest/task_validity.go b/chasm/lib/scheduler/schedulertest/task_validity.go index 4982a161770..72f4bb21be3 100644 --- a/chasm/lib/scheduler/schedulertest/task_validity.go +++ b/chasm/lib/scheduler/schedulertest/task_validity.go @@ -16,22 +16,20 @@ func (d *Driver) validIdleTask(s *scheduler.Scheduler, ctx chasm.Context) bool { handler := scheduler.NewSchedulerIdleTaskHandler(scheduler.SchedulerIdleTaskHandlerOptions{ Config: d.Config, MetricsHandler: metrics.NoopMetricsHandler, BaseLogger: d.Logger, }) - for _, node := range d.node().Snapshot(nil).Nodes { - for _, task := range node.GetMetadata().GetComponentAttributes().GetPureTasks() { - if task.GetTypeId() != id { - continue - } - value := &schedulerpb.SchedulerIdleTask{} - if err := proto.Unmarshal(task.GetData().GetData(), value); err != nil { - d.T.Fatalf("decode idle task: %v", err) - } - valid, err := handler.Validate(ctx, s, chasm.TaskInvocation{TaskAttributes: chasm.TaskAttributes{ScheduledTime: task.GetScheduledTime().AsTime()}}, value) - if err != nil { - d.T.Fatalf("validate idle task: %v", err) - } - if valid { - return true - } + for _, task := range d.runner.PendingPureTasks() { + if task.GetTypeId() != id { + continue + } + value := &schedulerpb.SchedulerIdleTask{} + if err := proto.Unmarshal(task.GetData().GetData(), value); err != nil { + d.T.Fatalf("decode idle task: %v", err) + } + valid, err := handler.Validate(ctx, s, chasm.TaskInvocation{TaskAttributes: chasm.TaskAttributes{ScheduledTime: task.GetScheduledTime().AsTime()}}, value) + if err != nil { + d.T.Fatalf("validate idle task: %v", err) + } + if valid { + return true } } return false diff --git a/chasm/lib/scheduler/schedulertest/task_validity_test.go b/chasm/lib/scheduler/schedulertest/task_validity_test.go index 232dea8e9e6..ce37d4ed648 100644 --- a/chasm/lib/scheduler/schedulertest/task_validity_test.go +++ b/chasm/lib/scheduler/schedulertest/task_validity_test.go @@ -21,14 +21,14 @@ func TestStalePhysicalTaskIsNotProgress(t *testing.T) { schedule.Spec.Interval = nil d := NewDriver(t, WithSchedule(schedule)) require.NoError(t, d.TriggerImmediately()) - require.NotEmpty(t, d.pendingSideEffectTasks()) + require.NotEmpty(t, d.runner.PendingSideEffectTasks()) _, err := d.Engine.UpdateComponent(chasm.NewEngineContext(context.Background(), d.Engine), d.ref, func(ctx chasm.MutableContext, c chasm.Component) error { c.(*scheduler.Scheduler).Invoker.Get(ctx).BufferedStarts = nil return nil }) require.NoError(t, err) - require.NotEmpty(t, d.pendingSideEffectTasks(), "physical deliveries survive logical invalidation") + require.NotEmpty(t, d.runner.PendingSideEffectTasks(), "physical deliveries survive logical invalidation") require.False(t, d.HasPendingTask()) require.False(t, d.StepSideEffect()) require.Empty(t, d.Snapshot().AcceptedStarts) @@ -41,7 +41,7 @@ func TestOldExecuteTaskCanProcessCurrentWork(t *testing.T) { schedule.Policies.OverlapPolicy = enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL d := NewDriver(t, WithSchedule(schedule)) require.NoError(t, d.TriggerImmediately()) - tasks := d.pendingSideEffectTasks() + tasks := d.runner.PendingSideEffectTasks() require.NotEmpty(t, tasks) old := tasks[0] _, err := d.Engine.UpdateComponent(chasm.NewEngineContext(context.Background(), d.Engine), d.ref, @@ -55,8 +55,7 @@ func TestOldExecuteTaskCanProcessCurrentWork(t *testing.T) { return nil }) require.NoError(t, err) - inTree, valid, err := d.node().ValidateSideEffectTask(context.Background(), old) - require.NoError(t, err) + inTree, valid := d.runner.ValidateSideEffectTask(old) require.True(t, inTree) require.True(t, valid) require.True(t, d.StepSideEffect()) From 75e20d0d59a16121aac78f47be3c6683f5a8b957 Mon Sep 17 00:00:00 2001 From: chaptersix <13949480+chaptersix@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:23:16 -0600 Subject: [PATCH 13/22] Port the scheduler mutation check to Go and discover property-test packages property-test now discovers every package whose tests import pgregory.net/rapid instead of hardcoding one directory, since -rapid.checks is undefined in other test binaries. The mutation check moves from Python to cmd/tools/mutationtest, runs via make mutation-test, and fails if any edit anchor does not match exactly once. The committed rapid failfile is removed (the same bug is pinned by TestLedger_FinalStartPendingAtIdle) and testdata/rapid is ignored. Co-Authored-By: Claude Fable 5.2 --- .gitignore | 4 + Makefile | 11 +- .../schedulertest/check_mutations.py | 50 ----- ...edulerProperties-20261111145706-76678.fail | 45 ---- cmd/tools/mutationtest/main.go | 195 ++++++++++++++++++ cmd/tools/mutationtest/main_test.go | 32 +++ 6 files changed, 241 insertions(+), 96 deletions(-) delete mode 100644 chasm/lib/scheduler/schedulertest/check_mutations.py delete mode 100644 chasm/lib/scheduler/schedulertest/testdata/rapid/TestSchedulerProperties/TestSchedulerProperties-20261111145706-76678.fail create mode 100644 cmd/tools/mutationtest/main.go create mode 100644 cmd/tools/mutationtest/main_test.go diff --git a/.gitignore b/.gitignore index f6e1955edc7..dd6703b24bf 100644 --- a/.gitignore +++ b/.gitignore @@ -48,3 +48,7 @@ # Ignoring AI agent files .agents/ + +# rapid failfiles replay raw draw sequences and go stale on generator changes; +# pin regressions with hand-written tests. CI uploads them as artifacts on failure. +**/testdata/rapid/ diff --git a/Makefile b/Makefile index 28b54fc2e12..de665b51608 100644 --- a/Makefile +++ b/Makefile @@ -519,7 +519,10 @@ leak-test: # Property tests get a dedicated run: more generated cases than the unit-test # default, and no retries, so a failing seed is reported instead of retried away. # Failing cases are minimized into testdata/rapid/ next to the test. -PROPERTY_TEST_DIRS ?= ./chasm/lib/scheduler/schedulertest/... +# Default: every package with a test file that imports pgregory.net/rapid (the +# -rapid.checks flag is only defined in test binaries that link rapid, so any +# other package would fail with "flag provided but not defined"). +PROPERTY_TEST_DIRS ?= $(patsubst %/,%,$(sort $(dir $(shell grep -l '"pgregory.net/rapid"' $(filter %_test.go,$(filter-out ./common/testx/% ./tests/mixedbrain/%,$(ALL_SRC))))))) PROPERTY_TEST_CHECKS ?= 5000 PROPERTY_TEST_TIMEOUT ?= 10m property-test: @@ -530,6 +533,12 @@ property-test: $(TEST_TAG_FLAG) \ -rapid.checks=$(PROPERTY_TEST_CHECKS) $(TEST_ARGS) +# Mutation checks: each mutates one production file via a go test overlay and +# requires the named regression test to fail. See cmd/tools/mutationtest. +mutation-test: + @printf $(COLOR) "Run mutation tests..." + CGO_ENABLED=$(TEST_CGO_ENABLED) go run ./cmd/tools/mutationtest + verify-test-log: @test -s test.log || (echo "TEST FAILURE: test.log is missing or empty" && exit 1) @grep -q "^ok" test.log || (echo "TEST FAILURE: no passing test found in test.log" && exit 1) diff --git a/chasm/lib/scheduler/schedulertest/check_mutations.py b/chasm/lib/scheduler/schedulertest/check_mutations.py deleted file mode 100644 index e62c6e868f9..00000000000 --- a/chasm/lib/scheduler/schedulertest/check_mutations.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python3 -"""Run targeted regressions against production mutations using Go overlays.""" -import json -import os -from pathlib import Path -import subprocess -import tempfile - -root = Path(__file__).resolve().parents[4] -mutations = [ - ("completion.go", "early-callback", "./chasm/lib/scheduler/schedulertest", - "TestStartAcceptanceCallbackAndRecording/AllowAll/before", - '\tif !internal.TracksCompletionResult(start.GetOverlapPolicy()) && start.GetRunId() == "" {', - '\tif start.GetCompleted() != nil {'), - ("invoker.go", "deferred-rearm", "./chasm/lib/scheduler/schedulertest", - "TestExpiredSelectionReleasesRemainingBuffer|TestLedger_DeferredStartsStrandedByPausedCompletion", - '\t// A discarded selection cannot produce', '\t// Re-arm tasks if this call changed state'), - ("backfiller_tasks.go", "capacity-progress", "./chasm/lib/scheduler", - "TestBackfillCapacityProgress", None, None), - ("invoker_tasks.go", "selected-start-reservation", "./chasm/lib/scheduler/schedulertest", - "TestSelectedStartReservesOverlapSlot", - "start.GetAttempt() > 0 && start.GetCompleted() == nil &&\n\t\t\t\t", - "internal.TracksCompletionResult(start.GetOverlapPolicy())\n\t\t})"), -] -for filename, name, package, test, begin, end in mutations: - source = root / "chasm/lib/scheduler" / filename - text = source.read_text() - if begin is not None: - first = text.index(begin) - last = text.index(end, first) - text = text[:first] + ("false && " if name == "selected-start-reservation" else "") + text[last:] - else: - original = "return min(available, max(1, available/backfillerCount))" - assert text.count(original) == 1 - text = text.replace(original, "return available / backfillerCount") - with tempfile.TemporaryDirectory(prefix="scheduler-mutation-") as temp: - directory = Path(temp) - replacement = directory / filename - replacement.write_text(text) - overlay = directory / "overlay.json" - overlay.write_text(json.dumps({"Replace": {str(source): str(replacement)}})) - result = subprocess.run( - ["go", "test", "-tags", "test_dep", "-overlay", str(overlay), package, - "-run", test, "-count=1"], - cwd=root, env={**os.environ, "CGO_ENABLED": "0"}, - text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - ) - if result.returncode == 0 or "--- FAIL: " not in result.stdout or "[build failed]" in result.stdout: - raise SystemExit(f"{name}: mutation did not produce an assertion failure\n{result.stdout}") - print(f"{name}: detected") diff --git a/chasm/lib/scheduler/schedulertest/testdata/rapid/TestSchedulerProperties/TestSchedulerProperties-20261111145706-76678.fail b/chasm/lib/scheduler/schedulertest/testdata/rapid/TestSchedulerProperties/TestSchedulerProperties-20261111145706-76678.fail deleted file mode 100644 index 9780b2dd1c5..00000000000 --- a/chasm/lib/scheduler/schedulertest/testdata/rapid/TestSchedulerProperties/TestSchedulerProperties-20261111145706-76678.fail +++ /dev/null @@ -1,45 +0,0 @@ -# 2026/11/11 14:57:06.863260 [TestSchedulerProperties] [rapid] draw idleTime: 600000000000 -# 2026/11/11 14:57:06.863265 [TestSchedulerProperties] [rapid] draw intervalSec: 10 -# 2026/11/11 14:57:06.863266 [TestSchedulerProperties] [rapid] draw jitterSec: 0 -# 2026/11/11 14:57:06.863266 [TestSchedulerProperties] [rapid] draw catchupSec: 60 -# 2026/11/11 14:57:06.863266 [TestSchedulerProperties] [rapid] draw overlapPolicy: 3 -# 2026/11/11 14:57:06.863267 [TestSchedulerProperties] [rapid] draw shape: "endTime" -# 2026/11/11 14:57:06.863268 [TestSchedulerProperties] [rapid] draw firesBeforeEnd: 3 -# 2026/11/11 14:57:06.863268 [TestSchedulerProperties] [rapid] draw paused: false -# 2026/11/11 14:57:06.863268 [TestSchedulerProperties] [rapid] draw limited: false -# 2026/11/11 14:57:06.863356 [TestSchedulerProperties] [rapid] draw numOps: 4 -# 2026/11/11 14:57:06.863357 [TestSchedulerProperties] [rapid] draw op: "step" -# 2026/11/11 14:57:06.863449 [TestSchedulerProperties] [rapid] draw op: "step" -# 2026/11/11 14:57:06.863489 [TestSchedulerProperties] [rapid] draw op: "step" -# 2026/11/11 14:57:06.863529 [TestSchedulerProperties] [rapid] draw op: "pure" -# 2026/11/11 14:57:06.863615 [TestSchedulerProperties] scheduler invariant violation at step (now=2020-01-01T00:10:30): [ledger-lost-start] scheduler closed with 1 buffered start(s) never sent to StartWorkflowExecution: [sched-auto-5ff7b19b-8077-5eee-9c97-bd1c5f9f67b6] -# -v0.4.8#7487548259739501542 -0x0 -0x0 -0x0 -0x0 -0x0 -0x0 -0x0 -0x0 -0x38e38e38e38e4 -0x2 -0x0 -0x1 -0x0 -0x38e38e38e38e4 -0x2 -0x0 -0x0 -0x0 -0x38e38e38e38e4 -0x3 -0x0 -0x0 -0x0 -0x0 -0x0 -0x0 -0x38e38e38e38e4 -0x2 diff --git a/cmd/tools/mutationtest/main.go b/cmd/tools/mutationtest/main.go new file mode 100644 index 00000000000..366d51b581b --- /dev/null +++ b/cmd/tools/mutationtest/main.go @@ -0,0 +1,195 @@ +// mutationtest checks that targeted regression tests detect deliberate +// production-code mutations. For each mutation it builds a mutated copy of one +// production file, points `go test -overlay` at it, runs the named tests, and +// requires a genuine assertion failure (not a build failure, not a pass). +// +// Edits are described by anchors that must each match exactly once in the +// unmutated file, so a refactor that moves or renames the anchored code makes +// this tool fail loudly instead of silently testing nothing. +// +// Run from anywhere inside the repository: go run ./cmd/tools/mutationtest +package main + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" +) + +// edit describes a single textual change to a file. +// +// If End is empty, the text Begin is replaced by Replacement. +// Otherwise the region from the start of Begin up to (but not including) the +// start of End is replaced by Replacement; End must appear after Begin. +// Begin and End must each occur exactly once in the file. +type edit struct { + Begin string + End string + Replacement string +} + +type mutation struct { + Name string + File string // repo-relative path of the production file to mutate + Package string // package passed to go test + Test string // -run regex + Edit edit +} + +const ( + schedulerDir = "chasm/lib/scheduler/" + schedulerTest = "./chasm/lib/scheduler/schedulertest" +) + +var mutations = []mutation{ + { + // Replace the guard so the callback is attached before a RunId exists. + Name: "early-callback", + File: schedulerDir + "completion.go", + Package: schedulerTest, + Test: "TestStartAcceptanceCallbackAndRecording/AllowAll/before", + Edit: edit{ + Begin: "\tif !internal.TracksCompletionResult(start.GetOverlapPolicy()) && start.GetRunId() == \"\" {", + Replacement: "\tif start.GetCompleted() != nil {", + }, + }, + { + // Delete the code between the discarded-selection comment and the + // re-arm comment, so deferred starts are never re-armed. + Name: "deferred-rearm", + File: schedulerDir + "invoker.go", + Package: schedulerTest, + Test: "TestExpiredSelectionReleasesRemainingBuffer|TestLedger_DeferredStartsStrandedByPausedCompletion", + Edit: edit{ + Begin: "\t// A discarded selection cannot produce", + End: "\t// Re-arm tasks if this call changed state", + }, + }, + { + Name: "capacity-progress", + File: schedulerDir + "backfiller_tasks.go", + Package: "./chasm/lib/scheduler", + Test: "TestBackfillCapacityProgress", + Edit: edit{ + Begin: "return min(available, max(1, available/backfillerCount))", + Replacement: "return available / backfillerCount", + }, + }, + { + // Short-circuit the selected-start predicate so that selected starts + // awaiting a RunId no longer hold the overlap slot. + Name: "selected-start-reservation", + File: schedulerDir + "invoker_tasks.go", + Package: schedulerTest, + Test: "TestSelectedStartReservesOverlapSlot", + Edit: edit{ + Begin: "start.GetAttempt() > 0 && start.GetCompleted() == nil &&\n\t\t\t\t", + End: "internal.TracksCompletionResult(start.GetOverlapPolicy())\n\t\t})", + Replacement: "false && ", + }, + }, +} + +func main() { + if err := run(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run() error { + root, err := repoRoot() + if err != nil { + return err + } + for _, m := range mutations { + if err := runMutation(root, m); err != nil { + return fmt.Errorf("%s: %w", m.Name, err) + } + fmt.Printf("%s: detected\n", m.Name) + } + return nil +} + +func repoRoot() (string, error) { + dir, err := os.Getwd() + if err != nil { + return "", err + } + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return dir, nil + } + parent := filepath.Dir(dir) + if parent == dir { + return "", errors.New("go.mod not found above working directory") + } + dir = parent + } +} + +// apply returns text with e applied, or an error if any anchor does not match +// exactly once. +func apply(text string, e edit) (string, error) { + if n := strings.Count(text, e.Begin); n != 1 { + return "", fmt.Errorf("begin anchor matched %d times, want exactly 1: %q", n, e.Begin) + } + first := strings.Index(text, e.Begin) + if e.End == "" { + return text[:first] + e.Replacement + text[first+len(e.Begin):], nil + } + if n := strings.Count(text, e.End); n != 1 { + return "", fmt.Errorf("end anchor matched %d times, want exactly 1: %q", n, e.End) + } + last := strings.Index(text, e.End) + if last < first { + return "", fmt.Errorf("end anchor %q appears before begin anchor %q", e.End, e.Begin) + } + return text[:first] + e.Replacement + text[last:], nil +} + +func runMutation(root string, m mutation) error { + source := filepath.Join(root, m.File) + original, err := os.ReadFile(source) + if err != nil { + return err + } + mutated, err := apply(string(original), m.Edit) + if err != nil { + return fmt.Errorf("%s: %w", m.File, err) + } + + temp, err := os.MkdirTemp("", "scheduler-mutation-") + if err != nil { + return err + } + defer func() { _ = os.RemoveAll(temp) }() + + replacement := filepath.Join(temp, filepath.Base(m.File)) + if err := os.WriteFile(replacement, []byte(mutated), 0o644); err != nil { + return err + } + overlay, err := json.Marshal(map[string]map[string]string{"Replace": {source: replacement}}) + if err != nil { + return err + } + overlayPath := filepath.Join(temp, "overlay.json") + if err := os.WriteFile(overlayPath, overlay, 0o644); err != nil { + return err + } + + cmd := exec.Command("go", "test", "-tags", "test_dep", "-overlay", overlayPath, + m.Package, "-run", m.Test, "-count=1") + cmd.Dir = root + cmd.Env = append(os.Environ(), "CGO_ENABLED=0") + out, err := cmd.CombinedOutput() + output := string(out) + if err == nil || !strings.Contains(output, "--- FAIL: ") || strings.Contains(output, "[build failed]") { + return fmt.Errorf("mutation did not produce an assertion failure\n%s", output) + } + return nil +} diff --git a/cmd/tools/mutationtest/main_test.go b/cmd/tools/mutationtest/main_test.go new file mode 100644 index 00000000000..d918765e43b --- /dev/null +++ b/cmd/tools/mutationtest/main_test.go @@ -0,0 +1,32 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestApply(t *testing.T) { + const text = "a one b two c" + + got, err := apply(text, edit{Begin: "one", Replacement: "1"}) + require.NoError(t, err) + require.Equal(t, "a 1 b two c", got) + + got, err = apply(text, edit{Begin: "one", End: "two", Replacement: "X "}) + require.NoError(t, err) + require.Equal(t, "a X two c", got) + + got, err = apply(text, edit{Begin: "one", End: "two"}) + require.NoError(t, err) + require.Equal(t, "a two c", got) + + _, err = apply(text, edit{Begin: "missing"}) + require.Error(t, err) + _, err = apply("x x", edit{Begin: "x"}) + require.Error(t, err) + _, err = apply(text, edit{Begin: "one", End: "absent"}) + require.Error(t, err) + _, err = apply(text, edit{Begin: "two", End: "one"}) + require.Error(t, err) +} From ef5690af3722820265f5070274008d07ce65f3cf Mon Sep 17 00:00:00 2001 From: chaptersix <13949480+chaptersix@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:24:42 -0600 Subject: [PATCH 14/22] Skip the unresolved migration counterexamples instead of gating them by environment The four counterexamples are known failures that need an unbuilt handoff protocol. An unconditional t.Skip naming the missing piece replaces the TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES gate, and the coverage plan now describes mutation-test and the uncommitted failfile. Co-Authored-By: Claude Fable 5.2 --- .../chasm-schedulertest-coverage-plan.md | 22 +++++++++---------- .../frontend/migration_delete_race_test.go | 5 +---- .../scheduler/migration_scenarios_test.go | 14 +++--------- 3 files changed, 14 insertions(+), 27 deletions(-) diff --git a/docs/architecture/chasm-schedulertest-coverage-plan.md b/docs/architecture/chasm-schedulertest-coverage-plan.md index 477d8eb8984..b9d6346a55b 100644 --- a/docs/architecture/chasm-schedulertest-coverage-plan.md +++ b/docs/architecture/chasm-schedulertest-coverage-plan.md @@ -22,7 +22,7 @@ authorship. The following signed commit contains the additions described here. | [#59](https://github.com/chaptersix/temporal/pull/59), [#62](https://github.com/chaptersix/temporal/pull/62) | Controlled acceptance/response events, buffer reordering during an outstanding RPC, repeated result reconciliation, and directed selected-start reservation regressions with a minimal reservation fix. Workflow scenarios are adapted without importing experimental activity APIs or action schemas. | | [#70](https://github.com/chaptersix/temporal/pull/70) | Duplicate callbacks before/after local start recording for ALLOW_ALL, SKIP and BUFFER_ALL, plus the early ALLOW_ALL callback guard. This branch retains callback attachment; the separate rollout-dependent callback-omission change is not imported. | | [#44](https://github.com/chaptersix/temporal/pull/44), [#45](https://github.com/chaptersix/temporal/pull/45) | `migration_handoff_test.go` and the V1 SDK workflow scenarios exercise lost responses, source-close boundaries, destination collisions and signal admission. | -| [#56](https://github.com/chaptersix/temporal/pull/56) | `service/frontend/migration_delete_race_test.go` includes the native control and an opt-in held-create/delete counterexample. V1 termination is mocked. | +| [#56](https://github.com/chaptersix/temporal/pull/56) | `service/frontend/migration_delete_race_test.go` includes the native control and a skipped held-create/delete counterexample. V1 termination is mocked. | | [#57](https://github.com/chaptersix/temporal/pull/57) | `migration_chain_ownership_test.go` adapts owned-descendant, foreign-chain and expired-first-run ownership checks to this baseline, with the persisted request ID and first-run ownership check. Receipt and start-pending persistence are not imported because they make a pre-commit Unavailable permanent. | ## Driver changes @@ -55,7 +55,7 @@ Go `go1.27.3 darwin/arm64` and Rapid `v1.3.0`. ```sh CGO_ENABLED=0 go test -tags test_dep ./chasm/chasmtest/... ./chasm/lib/scheduler/... ./service/frontend ./service/worker/scheduler -count=1 CGO_ENABLED=0 make property-test TEST_RACE_FLAG=off -python3 chasm/lib/scheduler/schedulertest/check_mutations.py +make mutation-test CGO_ENABLED=0 make lint-code-fast GOLANGCI_LINT_BASE_REV=03ecd1d228d2b8d9d433da55785afc9aeeead918 GOLANGCI_LINT_FIX=false ``` @@ -72,8 +72,9 @@ The original early callback run failed on `accepted-start-lost`, with ActionCoun still zero after external acceptance. The fix preserves exactly one counted start. The catchup-expiry failure is retained both as -`TestExpiredSelectionReleasesRemainingBuffer` and a Rapid failfile under -`schedulertest/testdata/rapid`. The explicit trace is: fixed initial clock +`TestExpiredSelectionReleasesRemainingBuffer`. The Rapid failfile is not +committed: it replays raw draws, which any generator change invalidates, and +`testdata/rapid` is ignored. The explicit trace is: fixed initial clock 2020-01-01 UTC; 10-second interval; end after 31 seconds; BUFFER_ALL; 60-second catchup; 10-minute idle; 5-minute workflow-duration bound; `step, step, step, pure`. The seeded replay is: @@ -117,12 +118,9 @@ Each fix has a default-run regression; none is gated. ## Unresolved counterexamples and limits -Four migration cases remain evidence, not fixes. Enable their intended -correctness assertions explicitly; this command is expected to fail: - -```sh -TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES=1 CGO_ENABLED=0 go test -tags test_dep ./service/frontend ./service/worker/scheduler -run TestMigrationCounterexample -count=1 -``` +Four migration cases remain evidence, not fixes. Each is an unconditional +`t.Skip` naming the missing handoff protocol; remove the skip to see the +assertion fail. - `TestMigrationCounterexample_TriggerDuringSnapshot`: a signal accepted while the migrate activity is pending is dropped with the old snapshot. @@ -153,8 +151,8 @@ admission fencing remain separate integration coverage. No new in-memory helper is presented as persistence or crash evidence. The source-close fault wrapper only tests transaction outcome/response ordering in the component engine. -The package tests above, `make property-test` at 5,000 cases, and the four gated -counterexamples also run under the race detector with no data race reported. +The package tests above and `make property-test` at 5,000 cases also run under +the race detector with no data race reported. On a macOS host whose default SDK is newer than its linker, the race build fails with `unknown architecture arm64e.x1`; set `SDKROOT` to the SDK matching the installed Command Line Tools. diff --git a/service/frontend/migration_delete_race_test.go b/service/frontend/migration_delete_race_test.go index 4efd279255c..80ce9250d57 100644 --- a/service/frontend/migration_delete_race_test.go +++ b/service/frontend/migration_delete_race_test.go @@ -3,7 +3,6 @@ package frontend import ( "context" "errors" - "os" "sync" "testing" "time" @@ -43,9 +42,7 @@ func (c *migrationDeleteRaceClient) DeleteSchedule(_ context.Context, req *sched func TestMigrationDeleteNativeControl(t *testing.T) { testMigrationDeleteRace(t, false) } func TestMigrationCounterexample_DeleteDuringCreate(t *testing.T) { - if os.Getenv("TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES") != "1" { - t.Skip("known migration counterexample; set TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES=1 to run") - } + t.Skip("A create held across DeleteSchedule commits an active scheduler afterwards; this needs a handoff protocol that records the delete as a tombstone the in-flight migration create must observe before committing.") testMigrationDeleteRace(t, true) } diff --git a/service/worker/scheduler/migration_scenarios_test.go b/service/worker/scheduler/migration_scenarios_test.go index 9f8b92f4d66..0a6a64ec63f 100644 --- a/service/worker/scheduler/migration_scenarios_test.go +++ b/service/worker/scheduler/migration_scenarios_test.go @@ -3,7 +3,6 @@ package scheduler import ( "context" "fmt" - "os" "testing" "time" @@ -49,19 +48,12 @@ func newMigrationScenario(t *testing.T) (*testsuite.TestWorkflowEnvironment, *sc } } -func requireMigrationCounterexamples(t *testing.T) { - t.Helper() - if os.Getenv("TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES") != "1" { - t.Skip("known migration counterexample; set TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES=1 to run") - } -} - func TestMigrationScenario_TriggerBeforeSnapshot(t *testing.T) { checkMigrationSignalAccounting(t, false) } func TestMigrationCounterexample_TriggerDuringSnapshot(t *testing.T) { - requireMigrationCounterexamples(t) + t.Skip("A trigger accepted while the migrate activity is pending is lost with the stale snapshot; this needs a handoff protocol that fences source signal admission once the snapshot is taken or forwards late signals to the destination.") checkMigrationSignalAccounting(t, true) } @@ -155,7 +147,7 @@ func TestMigrationScenario_RollbackFreshBackfill(t *testing.T) { } func TestMigrationCounterexample_ForwardDestinationCollision(t *testing.T) { - requireMigrationCounterexamples(t) + t.Skip("The forward activity treats AlreadyExists as success for an unrelated CHASM schedule; this needs a migration identity on the destination so the source can tell its own earlier create from a foreign schedule.") env, args := newMigrationScenario(t) when := timestamppb.New(baseStartTime) args.State.BufferedStarts = []*schedulespb.BufferedStart{{ @@ -209,7 +201,7 @@ func checkRollbackBackfillStart(t *testing.T, rollback bool) { } func TestMigrationCounterexample_LostCreateResponse(t *testing.T) { - requireMigrationCounterexamples(t) + t.Skip("After a lost create response the V1 source keeps executing work the committed destination also owns; this needs a handoff protocol that fences source execution until the destination commit is confirmed, gated on a new workflow version.") for _, version := range []SchedulerWorkflowVersion{TriggerImmediatelyTimestamp, MigrationHandoffFixes} { for _, disable := range []bool{false, true} { t.Run(fmt.Sprintf("version_%d_disable_%t", version, disable), func(t *testing.T) { From 4982ed7f70b3d1cc6d169893b5bd636c606913c7 Mon Sep 17 00:00:00 2001 From: chaptersix <13949480+chaptersix@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:32:03 -0600 Subject: [PATCH 15/22] Tidy the scheduler Snapshot and invariant helpers Remove the always-true IdleRouteChecked flag and AcceptedStarts nil guard, share one violation accumulator across the checks, and report every violation from CheckInvariantsHook instead of only the first. Co-Authored-By: Claude Fable 5.2 --- chasm/lib/scheduler/schedulertest/driver.go | 4 +- .../lib/scheduler/schedulertest/invariants.go | 91 ++++++++++--------- 2 files changed, 47 insertions(+), 48 deletions(-) diff --git a/chasm/lib/scheduler/schedulertest/driver.go b/chasm/lib/scheduler/schedulertest/driver.go index 7ed462e163c..f2554b9ba91 100644 --- a/chasm/lib/scheduler/schedulertest/driver.go +++ b/chasm/lib/scheduler/schedulertest/driver.go @@ -571,7 +571,6 @@ func (d *Driver) MigrateToWorkflow() error { // step's CloseTransaction). It is the input to invariant checks. type Snapshot struct { UntransferableActions int - IdleRouteChecked bool HasValidIdleTask bool AcceptedStarts map[string]ExternalStart RecordedStarts map[string]ExternalStart @@ -635,8 +634,7 @@ func (d *Driver) Snapshot() Snapshot { d.observeRecordedStarts() tweakables := d.Config.Tweakables(Namespace) snap := Snapshot{ - IdleRouteChecked: true, - AcceptedStarts: d.external.snapshot(), RecordedStarts: maps.Clone(d.recorded), + AcceptedStarts: d.external.snapshot(), RecordedStarts: maps.Clone(d.recorded), InitialActionCount: d.initialActionCount, Now: d.Now(), HasPendingTask: d.HasPendingTask(), diff --git a/chasm/lib/scheduler/schedulertest/invariants.go b/chasm/lib/scheduler/schedulertest/invariants.go index 537001b6b8c..09446ef5091 100644 --- a/chasm/lib/scheduler/schedulertest/invariants.go +++ b/chasm/lib/scheduler/schedulertest/invariants.go @@ -4,6 +4,7 @@ import ( "fmt" "maps" "slices" + "strings" ) // Violation describes a single broken invariant. @@ -16,6 +17,13 @@ func (v Violation) String() string { return fmt.Sprintf("[%s] %s", v.Name, v.Message) } +// violations accumulates the broken invariants found by a check. +type violations []Violation + +func (vs *violations) add(name, format string, args ...any) { + *vs = append(*vs, Violation{Name: name, Message: fmt.Sprintf(format, args...)}) +} + // CheckInvariants evaluates the scheduler invariants against a settled state // (cur). When prev is non-nil it also checks cross-step invariants (monotonicity, // closed-is-terminal). It returns all violations found; an empty slice means the @@ -26,17 +34,14 @@ func (v Violation) String() string { // only when it is held open (paused or draining a backfill). Any other taskless, // non-closed state is the "stuck" bug class. func CheckInvariants(prev *Snapshot, cur Snapshot) []Violation { - var violations []Violation - add := func(name, format string, args ...any) { - violations = append(violations, Violation{Name: name, Message: fmt.Sprintf(format, args...)}) - } + var vs violations // (1) No-stuck (primary): a live schedule must always have a way to make // progress. The only taskless, non-closed state allowed is "held open" // (paused or pending backfill). When idle is armed there is a pending idle // task, so HasPendingTask covers that case. if !cur.Closed && !cur.HasPendingTask && !cur.IsHeldOpen { - add("no-stuck", + vs.add("no-stuck", "scheduler has no pending task, is not closed, and is not held open "+ "(paused=%v backfillers=%d) at now=%s — stuck", cur.Paused, cur.BackfillerCount, cur.Now.Format("2006-01-02T15:04:05")) @@ -45,7 +50,7 @@ func CheckInvariants(prev *Snapshot, cur Snapshot) []Violation { // (2) Idle correctness: a held-open schedule must never have an idle-close // armed; idle close would race the customer's intent to resume/drain. if cur.IsHeldOpen && !cur.IdleCloseTime.IsZero() { - add("idle-not-while-held-open", + vs.add("idle-not-while-held-open", "scheduler is held open (paused=%v backfillers=%d) but has idle close armed at %s", cur.Paused, cur.BackfillerCount, cur.IdleCloseTime.Format("2006-01-02T15:04:05")) } @@ -55,7 +60,7 @@ func CheckInvariants(prev *Snapshot, cur Snapshot) []Violation { // len(buffered)); exceeding it means the limit was not enforced and CHASM state // can grow without bound (oversized state, persistence failures, OOM). if cur.MaxBufferSize > 0 && cur.BufferedStartsCount > cur.MaxBufferSize { - add("buffer-bound", + vs.add("buffer-bound", "buffered starts = %d, exceeding MaxBufferSize=%d", cur.BufferedStartsCount, cur.MaxBufferSize) } @@ -66,39 +71,39 @@ func CheckInvariants(prev *Snapshot, cur Snapshot) []Violation { // reset across the terminate/cancel/start phases, letting one execution take up // to a multiple of the configured maximum. if cur.MaxActionsPerExecution > 0 && cur.MaxActionsObserved > cur.MaxActionsPerExecution { - add("action-budget", + vs.add("action-budget", "a single ExecuteTask took %d actions, exceeding MaxActionsPerExecution=%d", cur.MaxActionsObserved, cur.MaxActionsPerExecution) } - violations = append(violations, checkLedger(prev, cur)...) - violations = append(violations, checkStartAccounting(cur)...) - if cur.IdleRouteChecked && !cur.Closed && !cur.IdleCloseTime.IsZero() && !cur.HasValidIdleTask { - add("idle-route", "advertised idle deadline %s has no valid logical idle task", cur.IdleCloseTime) + vs = append(vs, checkLedger(prev, cur)...) + vs = append(vs, checkStartAccounting(cur)...) + if !cur.Closed && !cur.IdleCloseTime.IsZero() && !cur.HasValidIdleTask { + vs.add("idle-route", "advertised idle deadline %s has no valid logical idle task", cur.IdleCloseTime) } if prev != nil { // (3) Closed is terminal: a closed schedule must never reopen. if prev.Closed && !cur.Closed { - add("closed-terminal", "scheduler reopened after being closed") + vs.add("closed-terminal", "scheduler reopened after being closed") } // (4) High-water-mark monotonicity: generator/invoker HWMs never regress. if cur.GeneratorLPT.Before(prev.GeneratorLPT) { - add("hwm-monotonic-generator", + vs.add("hwm-monotonic-generator", "generator high-water mark went backward: %s -> %s", prev.GeneratorLPT.Format("2006-01-02T15:04:05"), cur.GeneratorLPT.Format("2006-01-02T15:04:05")) } if cur.InvokerLPT.Before(prev.InvokerLPT) { - add("hwm-monotonic-invoker", + vs.add("hwm-monotonic-invoker", "invoker high-water mark went backward: %s -> %s", prev.InvokerLPT.Format("2006-01-02T15:04:05"), cur.InvokerLPT.Format("2006-01-02T15:04:05")) } } - return violations + return vs } // fataler is the subset of testing.TB the invariant assertion helper needs. @@ -108,7 +113,7 @@ type fataler interface { } // CheckInvariantsHook returns an AfterStep hook that captures a snapshot after -// every step and fails the test on the first invariant violation. It threads the +// every step and fails the test, reporting every invariant violation found. It threads the // previous snapshot so cross-step invariants are checked. This is the normal way // invariant checking is wired into a Driver: // @@ -119,9 +124,13 @@ func CheckInvariantsHook(t fataler) func(d *Driver) { return func(d *Driver) { t.Helper() cur := d.Snapshot() - if violations := CheckInvariants(prev, cur); len(violations) > 0 { + if vs := CheckInvariants(prev, cur); len(vs) > 0 { + msgs := make([]string, len(vs)) + for i, v := range vs { + msgs[i] = v.String() + } t.Fatalf("scheduler invariant violation at step (now=%s): %s", - cur.Now.Format("2006-01-02T15:04:05"), violations[0]) + cur.Now.Format("2006-01-02T15:04:05"), strings.Join(msgs, "; ")) } snap := cur prev = &snap @@ -131,27 +140,24 @@ func CheckInvariantsHook(t fataler) func(d *Driver) { // checkLedger checks that every start the scheduler buffered is either sent to // StartWorkflowExecution or discarded on purpose, and never lost to a close. func checkLedger(prev *Snapshot, cur Snapshot) []Violation { - var violations []Violation - add := func(name, format string, args ...any) { - violations = append(violations, Violation{Name: name, Message: fmt.Sprintf(format, args...)}) - } + var vs violations if cur.Migrating && cur.UntransferableActions > 0 { - add("migration-untransferable", "%d execution/retry/cancel/terminate obligations lost native task ownership", cur.UntransferableActions) + vs.add("migration-untransferable", "%d execution/retry/cancel/terminate obligations lost native task ownership", cur.UntransferableActions) } if cur.Migrating && cur.Closed && len(cur.PendingStarts) > 0 { - add("migration-transfer-unverified", "closed source retains %d obligations without a verified destination", len(cur.PendingStarts)) + vs.add("migration-transfer-unverified", "closed source retains %d obligations without a verified destination", len(cur.PendingStarts)) } // Terminal half: a schedule never closes while a start it buffered is // still waiting for InvokerExecuteTask (temporal#13016). if cur.Closed && !cur.Migrating && len(cur.PendingStarts) > 0 { - add("ledger-lost-start", + vs.add("ledger-lost-start", "scheduler closed with %d buffered start(s) never sent to StartWorkflowExecution: %v", len(cur.PendingStarts), slices.Sorted(maps.Keys(cur.PendingStarts))) } if prev == nil { - return violations + return vs } // Conservation half: every start that leaves the buffer without a RunId was @@ -170,31 +176,26 @@ func checkLedger(prev *Snapshot, cur Snapshot) []Violation { discarded := (cur.OverlapSkipped - prev.OverlapSkipped) + (cur.DroppedStarts - prev.DroppedStarts) if int64(len(silent)) > discarded { slices.Sort(silent) - add("ledger-silent-drop", + vs.add("ledger-silent-drop", "%d buffered start(s) left the buffer without a start RPC, but only %d discard(s) were recorded: %v", len(silent), discarded, silent) } - return violations + return vs } func checkStartAccounting(cur Snapshot) []Violation { - var violations []Violation - add := func(name, format string, args ...any) { - violations = append(violations, Violation{Name: name, Message: fmt.Sprintf(format, args...)}) - } - if cur.AcceptedStarts != nil { - for id, accepted := range cur.AcceptedStarts { - recorded, ok := cur.RecordedStarts[id] - if ok && recorded != accepted { - add("start-identity", "request %s recorded %+v, accepted %+v", id, recorded, accepted) - } - if _, pending := cur.PendingStarts[id]; !ok && !pending { - add("accepted-start-lost", "accepted request %s has neither a pending obligation nor a recorded result", id) - } + var vs violations + for id, accepted := range cur.AcceptedStarts { + recorded, ok := cur.RecordedStarts[id] + if ok && recorded != accepted { + vs.add("start-identity", "request %s recorded %+v, accepted %+v", id, recorded, accepted) } - if want := cur.InitialActionCount + int64(len(cur.RecordedStarts)); cur.ActionCount != want { - add("start-accounting", "ActionCount=%d, observed recorded starts plus initial count=%d", cur.ActionCount, want) + if _, pending := cur.PendingStarts[id]; !ok && !pending { + vs.add("accepted-start-lost", "accepted request %s has neither a pending obligation nor a recorded result", id) } } - return violations + if want := cur.InitialActionCount + int64(len(cur.RecordedStarts)); cur.ActionCount != want { + vs.add("start-accounting", "ActionCount=%d, observed recorded starts plus initial count=%d", cur.ActionCount, want) + } + return vs } From 7cd334167cdaa851393e2dcc7102ed0def8814b4 Mon Sep 17 00:00:00 2001 From: chaptersix <13949480+chaptersix@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:48:11 -0600 Subject: [PATCH 16/22] Drive the scheduler property as a state machine with wider operations TestSchedulerProperties now runs as a rapid state machine over step, pure, sideEffect, complete, pause, unpause, trigger, backfill, update, failed starts and lost start responses, with small drawn buffer and action limits and the CANCEL_OTHER and TERMINATE_OTHER overlap policies. Harness fixes the wider generator needed: the retry policy is now free of random jitter so rapid can replay and shrink, the task runner no longer jumps the clock over work its BeforeAdvance hook schedules earlier, and the start ledger accepts a start replaced by a CANCEL_OTHER or TERMINATE_OTHER start. Two production bugs it found are recorded as skipped reproducers. Co-Authored-By: Claude Fable 5.2 --- chasm/chasmtest/task_runner.go | 26 ++- chasm/chasmtest/task_runner_test.go | 21 ++ chasm/lib/scheduler/schedulertest/driver.go | 112 ++++++++++- .../lib/scheduler/schedulertest/invariants.go | 7 +- .../schedulertest/known_bugs_test.go | 84 ++++++++ .../scheduler/schedulertest/ledger_test.go | 5 + chasm/lib/scheduler/schedulertest/library.go | 15 +- .../scheduler/schedulertest/property_test.go | 184 +++++++++++++----- 8 files changed, 394 insertions(+), 60 deletions(-) create mode 100644 chasm/lib/scheduler/schedulertest/known_bugs_test.go diff --git a/chasm/chasmtest/task_runner.go b/chasm/chasmtest/task_runner.go index 80fd4c68bf4..3aea3c0cc7e 100644 --- a/chasm/chasmtest/task_runner.go +++ b/chasm/chasmtest/task_runner.go @@ -65,7 +65,8 @@ type TaskRunnerHooks struct { // BeforeAdvance runs before the clock moves to the given time and may // itself move the clock to an earlier instant, for example to complete // external work that would otherwise run past a deadline. It runs even - // when the clock does not end up moving. + // when the clock does not end up moving. If it schedules work earlier than + // the given time, the runner stops there instead and calls it again. BeforeAdvance func(to time.Time) // BeforeSideEffect runs after a task has been selected and validated, just // before it executes. @@ -140,7 +141,7 @@ func (r *TaskRunner) Step() bool { if !ok { return false } - r.advanceTo(fireTime) + r.advanceTo(fireTime, r.nextFireTime) r.firePureTasks() for r.StepSideEffect() { } @@ -163,7 +164,7 @@ func (r *TaskRunner) StepPure() bool { fireTime.After(oldest.Add(r.maxSideEffectLag)) { return false } - r.advanceTo(fireTime) + r.advanceTo(fireTime, r.nextPureFireTime) r.firePureTasks() return true } @@ -477,10 +478,23 @@ func (r *TaskRunner) afterStep() { } // advanceTo moves the clock to t, never backward, after giving BeforeAdvance a -// chance to act on the way. -func (r *TaskRunner) advanceTo(t time.Time) { - if r.hooks.BeforeAdvance != nil { +// chance to act on the way. BeforeAdvance may schedule work that falls before +// t, such as the retry of a start it dispatched; next reports the earliest +// pending fire time, and the clock stops there instead of jumping over it. +func (r *TaskRunner) advanceTo(t time.Time, next func() (time.Time, bool)) { + for r.hooks.BeforeAdvance != nil { r.hooks.BeforeAdvance(t) + earliest, ok := next() + if !ok { + break + } + if now := r.clock.Now(); earliest.Before(now) { + earliest = now + } + if !earliest.Before(t) { + break + } + t = earliest } if t.After(r.clock.Now()) { r.clock.Update(t) diff --git a/chasm/chasmtest/task_runner_test.go b/chasm/chasmtest/task_runner_test.go index b02199d46f2..04716607616 100644 --- a/chasm/chasmtest/task_runner_test.go +++ b/chasm/chasmtest/task_runner_test.go @@ -217,6 +217,27 @@ func TestTaskRunnerBeforeAdvanceMayMoveClock(t *testing.T) { require.Equal(t, runnerStart.Add(time.Hour), f.ts.Now(), "the runner still lands on the task's time") } +func TestTaskRunnerBeforeAdvanceSchedulesEarlierWork(t *testing.T) { + f := newRunnerFixture(t, time.Hour) + var advances []time.Time + var dispatchedAt []time.Time + r := chasmtest.NewTaskRunner(t, f.engine, f.ref, f.ts, chasmtest.WithTaskRunnerHooks(chasmtest.TaskRunnerHooks{ + BeforeAdvance: func(to time.Time) { + advances = append(advances, to) + if len(advances) == 1 { + f.addSideEffect(t, "retry", 10*time.Minute, true) + } + }, + AfterSideEffect: func() { dispatchedAt = append(dispatchedAt, f.ts.Now()) }, + })) + + require.True(t, r.Step()) + require.Equal(t, []time.Time{runnerStart.Add(time.Hour), runnerStart.Add(10 * time.Minute)}, advances) + require.Equal(t, []time.Time{runnerStart.Add(10 * time.Minute)}, dispatchedAt, + "work the hook scheduled before the target time is not jumped over") + require.Equal(t, runnerStart.Add(10*time.Minute), f.ts.Now()) +} + func TestTaskRunnerToleratedSideEffectError(t *testing.T) { f := newRunnerFixture(t, 0) f.addSideEffect(t, "ghost", time.Minute, false) diff --git a/chasm/lib/scheduler/schedulertest/driver.go b/chasm/lib/scheduler/schedulertest/driver.go index f2554b9ba91..de5ef3f163a 100644 --- a/chasm/lib/scheduler/schedulertest/driver.go +++ b/chasm/lib/scheduler/schedulertest/driver.go @@ -1,6 +1,7 @@ package schedulertest import ( + "cmp" "context" "errors" "fmt" @@ -34,6 +35,12 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" ) +// startFaults counts the upcoming StartWorkflowExecution RPCs that fail. +type startFaults struct { + rejected int // the RPC fails without the start being accepted + lostReply int // the start is accepted but the response is lost +} + // Driver builds a real CHASM scheduler on top of a [chasmtest.Engine] and steps // it forward through every task it schedules for itself, advancing a controllable // clock. After each settled transition it invokes AfterStep (if set), which is @@ -50,6 +57,7 @@ type Driver struct { startCalls chan *StartCall actionsBefore int recorded map[string]ExternalStart + replacingSeen map[string]struct{} initialActionCount int64 T chasmtest.TB @@ -89,6 +97,9 @@ type Driver struct { // starts from concurrent goroutines. rpcMu *sync.Mutex + // faults injects failures into uncontrolled start RPCs. Guarded by rpcMu. + faults *startFaults + // maxRunDuration auto-completes a started workflow before the clock moves // past its StartTime plus this. Zero means workflows only complete through // CompleteWorkflow. @@ -214,6 +225,7 @@ func newDriverScaffold(t chasmtest.TB, opts ...DriverOption) (*Driver, *driverCo startCalls = make(chan *StartCall) } rpcMu := &sync.Mutex{} + faults := &startFaults{} countAction := func() { rpcMu.Lock() defer rpcMu.Unlock() @@ -241,10 +253,24 @@ func newDriverScaffold(t chasmtest.TB, opts ...DriverOption) (*Driver, *driverCo return nil, ctx.Err() } } + rpcMu.Lock() + rejected, lostReply := faults.rejected > 0, faults.rejected == 0 && faults.lostReply > 0 + if rejected { + faults.rejected-- + } else if lostReply { + faults.lostReply-- + } + rpcMu.Unlock() + if rejected { + return nil, serviceerror.NewUnavailable("start rejected by driver") + } accepted, err := external.accept(req) if err != nil { return nil, err } + if lostReply { + return nil, serviceerror.NewUnavailable("start accepted, response lost by driver") + } return &workflowservice.StartWorkflowExecutionResponse{RunId: accepted.RunID}, nil }).AnyTimes() history.EXPECT().TerminateWorkflowExecution(gomock.Any(), gomock.Any()). @@ -286,6 +312,7 @@ func newDriverScaffold(t chasmtest.TB, opts ...DriverOption) (*Driver, *driverCo d := &Driver{ recordCalls: make(chan *RecordCall), external: external, startCalls: startCalls, recorded: make(map[string]ExternalStart), + replacingSeen: make(map[string]struct{}), T: t, Engine: engine, Registry: registry, @@ -297,6 +324,7 @@ func newDriverScaffold(t chasmtest.TB, opts ...DriverOption) (*Driver, *driverCo actionRPCs: actionRPCs, startRequests: startRequests, rpcMu: rpcMu, + faults: faults, metricsCapture: capture, maxRunDuration: cfg.maxRun, AfterStep: cfg.afterStep, @@ -529,6 +557,60 @@ func (d *Driver) Patch(patch *schedulepb.SchedulePatch) error { return err } +// FailNextStarts makes the next n StartWorkflowExecution RPCs fail before the +// start is accepted, then lets later RPCs (the scheduler's retries) succeed. +// It does not apply under WithControlledStarts, where the test decides each +// RPC's outcome. +func (d *Driver) FailNextStarts(n int) { + d.rpcMu.Lock() + defer d.rpcMu.Unlock() + d.faults.rejected += n +} + +// LoseNextStartResponses makes the next n StartWorkflowExecution RPCs (after +// any FailNextStarts) accept the start but return an error, so the scheduler +// must retry with the same request ID and rely on deduplication. +func (d *Driver) LoseNextStartResponses(n int) { + d.rpcMu.Lock() + defer d.rpcMu.Unlock() + d.faults.lostReply += n +} + +// Backfill requests a backfill of the given range via Patch. +func (d *Driver) Backfill(req *schedulepb.BackfillRequest) error { + return d.Patch(&schedulepb.SchedulePatch{BackfillRequest: []*schedulepb.BackfillRequest{req}}) +} + +// Update applies mutate to a copy of the current schedule definition and +// replaces the schedule with it through the production Scheduler.Update path, +// without a conflict token. It returns the scheduler's error (e.g. ErrClosed) +// without failing the test. On success the AfterStep hook runs. +func (d *Driver) Update(mutate func(*schedulepb.Schedule)) error { + d.T.Helper() + engineCtx := chasm.NewEngineContext(context.Background(), d.Engine) + _, err := d.Engine.UpdateComponent(engineCtx, d.ref, func(mctx chasm.MutableContext, c chasm.Component) error { + s, ok := c.(*scheduler.Scheduler) + if !ok { + return fmt.Errorf("root component is %T, want *scheduler.Scheduler", c) + } + next := common.CloneProto(s.Schedule) + mutate(next) + _, uerr := s.Update(mctx, &schedulerpb.UpdateScheduleRequest{ + NamespaceId: NamespaceID, + FrontendRequest: &workflowservice.UpdateScheduleRequest{ + Namespace: Namespace, + ScheduleId: ScheduleID, + Schedule: next, + }, + }) + return uerr + }) + if err == nil && d.AfterStep != nil { + d.AfterStep(d) + } + return err +} + // Pause pauses the schedule via Patch. func (d *Driver) Pause() error { return d.Patch(&schedulepb.SchedulePatch{Pause: "paused by driver"}) @@ -617,8 +699,17 @@ type Snapshot struct { // metric for missed-catchup and paused/limited drops). PendingStarts map[string]struct{} StartRequested map[string]struct{} - OverlapSkipped int64 - DroppedStarts int64 + // ReplacingStarts counts the pending starts that no start RPC has been attempted for (including deferred ones) whose overlap policy + // is CANCEL_OTHER or TERMINATE_OTHER. With nothing running, such a start + // replaces the one the buffer would otherwise have started + // (legacyscheduler.ProcessBuffer), discarding it without counting an overlap + // skip, so each can account for one silent disappearance. ReplacingSeen is + // the number of distinct such starts ever seen in the buffer in any state, + // which covers a replacer that was buffered and started between two snapshots. + ReplacingStarts int + ReplacingSeen int + OverlapSkipped int64 + DroppedStarts int64 // Migrating is true once the schedule hands off to a V1 workflow, which // takes its buffered starts with it. Migrating bool @@ -667,7 +758,8 @@ func (d *Driver) Snapshot() Snapshot { if inv := s.Invoker.Get(ctx); inv != nil { snap.UntransferableActions = len(inv.GetCancelWorkflows()) + len(inv.GetTerminateWorkflows()) snap.BufferedStartsCount = len(inv.GetBufferedStarts()) - snap.addBufferedStarts(inv.GetBufferedStarts()) + snap.addBufferedStarts(inv.GetBufferedStarts(), s.Schedule.GetPolicies().GetOverlapPolicy(), d.replacingSeen) + snap.ReplacingSeen = len(d.replacingSeen) if inv.LastProcessedTime != nil { snap.InvokerLPT = inv.LastProcessedTime.AsTime() } @@ -678,11 +770,21 @@ func (d *Driver) Snapshot() Snapshot { } // addBufferedStarts classifies buffered starts into pending (deferred or not) -// and running. -func (snap *Snapshot) addBufferedStarts(starts []*schedulespb.BufferedStart) { +// and running. scheduleOverlap resolves starts with an unspecified policy. +// replacingSeen accumulates the request IDs of replacing starts across snapshots. +func (snap *Snapshot) addBufferedStarts(starts []*schedulespb.BufferedStart, scheduleOverlap enumspb.ScheduleOverlapPolicy, replacingSeen map[string]struct{}) { for _, start := range starts { + policy := cmp.Or(start.GetOverlapPolicy(), scheduleOverlap) + replacing := policy == enumspb.SCHEDULE_OVERLAP_POLICY_CANCEL_OTHER || + policy == enumspb.SCHEDULE_OVERLAP_POLICY_TERMINATE_OTHER + if replacing { + replacingSeen[start.GetRequestId()] = struct{}{} + } switch { case start.GetRunId() == "": + if replacing && start.GetAttempt() <= 0 { + snap.ReplacingStarts++ + } if start.GetAttempt() > 0 { snap.UntransferableActions++ } diff --git a/chasm/lib/scheduler/schedulertest/invariants.go b/chasm/lib/scheduler/schedulertest/invariants.go index 09446ef5091..394b31101ee 100644 --- a/chasm/lib/scheduler/schedulertest/invariants.go +++ b/chasm/lib/scheduler/schedulertest/invariants.go @@ -164,7 +164,9 @@ func checkLedger(prev *Snapshot, cur Snapshot) []Violation { // either sent to StartWorkflowExecution or discarded on purpose (overlap // skip, missed catchup, paused/limited drop). Starts that are buffered and // discarded within one transaction never appear pending, so the discard - // counters can only exceed the disappearances, never trail. + // counters can only exceed the disappearances, never trail. A start may also + // be replaced by a CANCEL_OTHER or TERMINATE_OTHER start (see + // Snapshot.ReplacingStarts). var silent []string for id := range prev.PendingStarts { _, still := cur.PendingStarts[id] @@ -173,7 +175,8 @@ func checkLedger(prev *Snapshot, cur Snapshot) []Violation { silent = append(silent, id) } } - discarded := (cur.OverlapSkipped - prev.OverlapSkipped) + (cur.DroppedStarts - prev.DroppedStarts) + discarded := (cur.OverlapSkipped - prev.OverlapSkipped) + (cur.DroppedStarts - prev.DroppedStarts) + + int64(prev.ReplacingStarts+cur.ReplacingSeen-prev.ReplacingSeen) if int64(len(silent)) > discarded { slices.Sort(silent) vs.add("ledger-silent-drop", diff --git a/chasm/lib/scheduler/schedulertest/known_bugs_test.go b/chasm/lib/scheduler/schedulertest/known_bugs_test.go new file mode 100644 index 00000000000..e4e91069f22 --- /dev/null +++ b/chasm/lib/scheduler/schedulertest/known_bugs_test.go @@ -0,0 +1,84 @@ +package schedulertest_test + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + enumspb "go.temporal.io/api/enums/v1" + schedulepb "go.temporal.io/api/schedule/v1" + "go.temporal.io/server/chasm/lib/scheduler" + "go.temporal.io/server/chasm/lib/scheduler/schedulertest" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// TestActionBudgetSharedAcrossPhases reproduces a bug found by +// TestSchedulerProperties. InvokerExecuteTask.Execute passes its +// invokerTaskHandlerContext by value to terminateWorkflows, cancelWorkflows and +// startWorkflows, so each phase spends from its own copy of the action budget. +// A cancel from a CANCEL_OTHER start and a terminate from a later +// TERMINATE_OTHER start both target the same running workflow, and one +// execution makes two RPCs although MaxActionsPerExecution is 1. +func TestActionBudgetSharedAcrossPhases(t *testing.T) { + t.Skip("InvokerExecuteTask spends the MaxActionsPerExecution budget separately in the terminate, cancel and start phases (invoker_tasks.go Execute passes ictx by value)") + + start := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) + s := schedulertest.DefaultSchedule() + s.Spec.Interval[0].Interval = durationpb.New(10 * time.Second) + s.Policies.OverlapPolicy = enumspb.SCHEDULE_OVERLAP_POLICY_SKIP + var vs []schedulertest.Violation + d := schedulertest.NewDriver(t, + schedulertest.WithStartTime(start), + schedulertest.WithSchedule(s), + schedulertest.WithConfig(configWith(func(tw *scheduler.Tweakables) { tw.MaxActionsPerExecution = 1 })), + schedulertest.WithAfterStep(collectViolations(&vs)), + ) + backfill := func(policy enumspb.ScheduleOverlapPolicy) { + at := timestamppb.New(start.Add(-time.Minute)) + require.NoError(t, d.Backfill(&schedulepb.BackfillRequest{StartTime: at, EndTime: at, OverlapPolicy: policy})) + } + + require.NoError(t, d.TriggerImmediately()) + backfill(enumspb.SCHEDULE_OVERLAP_POLICY_CANCEL_OTHER) + require.True(t, d.StepSideEffect(), "start the triggered workflow") + backfill(enumspb.SCHEDULE_OVERLAP_POLICY_TERMINATE_OTHER) + require.True(t, d.StepSideEffect(), "cancel and terminate the running workflow") + + require.Empty(t, violationNames(vs)) +} + +// TestBackfillRespectsSmallMaxBufferSize reproduces a bug found by +// TestSchedulerProperties. The backfiller's capacity calculation subtracts the +// constant recentActionCount (10) from the buffer length instead of the number +// of completed starts actually retained, so with a MaxBufferSize below 20 it +// keeps adding starts until the buffer holds about recentActionCount plus +// MaxBufferSize/2 entries. +func TestBackfillRespectsSmallMaxBufferSize(t *testing.T) { + t.Skip("backfillerBufferCapacity assumes recentActionCount completed starts are retained, so a backfill overfills a MaxBufferSize below 20 (backfiller_tasks.go allowedBufferedStarts)") + + start := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) + s := schedulertest.DefaultSchedule() + s.Policies.OverlapPolicy = enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ALL + s.State.Paused = true + var vs []schedulertest.Violation + d := schedulertest.NewDriver(t, + schedulertest.WithStartTime(start.Add(10*time.Minute)), + schedulertest.WithSchedule(s), + schedulertest.WithConfig(configWith(func(tw *scheduler.Tweakables) { + tw.MaxBufferSize = 2 + tw.GeneratorBufferReserveSize = 0 + })), + schedulertest.WithAfterStep(collectViolations(&vs)), + ) + require.NoError(t, d.Backfill(&schedulepb.BackfillRequest{ + StartTime: timestamppb.New(start), + EndTime: timestamppb.New(start.Add(10 * time.Minute)), + OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ALL, + })) + for range 12 { + d.Step() + } + + require.Empty(t, violationNames(vs)) +} diff --git a/chasm/lib/scheduler/schedulertest/ledger_test.go b/chasm/lib/scheduler/schedulertest/ledger_test.go index 397d85b0188..04b7465c848 100644 --- a/chasm/lib/scheduler/schedulertest/ledger_test.go +++ b/chasm/lib/scheduler/schedulertest/ledger_test.go @@ -131,6 +131,11 @@ func TestCheckInvariants_Ledger(t *testing.T) { prev: schedulertest.Snapshot{HasPendingTask: true, PendingStarts: pending("a", "b"), DroppedStarts: 3}, cur: schedulertest.Snapshot{HasPendingTask: true, DroppedStarts: 4, StartRequested: pending("b")}, }, + { + name: "replaced by cancel-other start", + prev: schedulertest.Snapshot{HasPendingTask: true, PendingStarts: pending("a", "b"), ReplacingStarts: 1}, + cur: schedulertest.Snapshot{HasPendingTask: true, StartRequested: pending("b")}, + }, { name: "silent drop", prev: schedulertest.Snapshot{HasPendingTask: true, PendingStarts: pending("a", "b")}, diff --git a/chasm/lib/scheduler/schedulertest/library.go b/chasm/lib/scheduler/schedulertest/library.go index e8b827e321b..8cbbdd18758 100644 --- a/chasm/lib/scheduler/schedulertest/library.go +++ b/chasm/lib/scheduler/schedulertest/library.go @@ -83,12 +83,21 @@ func DefaultConfig() *scheduler.Config { EncodeInternalTokenWithEnvelope: func(string) bool { return true }, - RetryPolicy: func() backoff.RetryPolicy { - return backoff.NewExponentialRetryPolicy(1 * time.Second) - }, + RetryPolicy: func() backoff.RetryPolicy { return deterministicRetryPolicy{} }, } } +// deterministicRetryPolicy backs off exponentially from one second to a minute +// like the production policy but without its random jitter, so that a rapid +// run replays identically while it shrinks a failure. +type deterministicRetryPolicy struct{} + +func (deterministicRetryPolicy) ComputeNextDelay(_ time.Duration, numAttempts int, _ error) time.Duration { + return min(time.Second< 0 { @@ -99,12 +200,7 @@ func drawSchedule(rt *rapid.T, start time.Time, idle time.Duration) *schedulepb. s.Policies.CatchupWindow = durationpb.New(time.Duration(catchupSec) * time.Second) } - s.Policies.OverlapPolicy = rapid.SampledFrom([]enumspb.ScheduleOverlapPolicy{ - enumspb.SCHEDULE_OVERLAP_POLICY_SKIP, - enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ONE, - enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ALL, - enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL, - }).Draw(rt, "overlapPolicy") + s.Policies.OverlapPolicy = drawOverlapPolicy(rt, "overlapPolicy", policies) switch rapid.SampledFrom([]string{"forever", "endTime", "oneShot"}).Draw(rt, "shape") { case "endTime": From 4a76fa7392ad37e5b04bd8e0ae0685ac796ca90c Mon Sep 17 00:00:00 2001 From: chaptersix <13949480+chaptersix@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:52:01 -0600 Subject: [PATCH 17/22] Share one action budget across the invoker's terminate, cancel and start phases InvokerExecuteTask passed its per-task context by value, so each phase spent MaxActionsPerExecution from its own copy and one execution could take up to three times the budget. Pass it by pointer. The widened scheduler property found this. Its reproducer is now unskipped and the generator draws every overlap policy with a small budget again. Co-Authored-By: Claude Fable 5.2 --- chasm/lib/scheduler/invoker_tasks.go | 9 +++++---- .../schedulertest/known_bugs_test.go | 12 +++++------ .../scheduler/schedulertest/property_test.go | 20 +++++-------------- 3 files changed, 15 insertions(+), 26 deletions(-) diff --git a/chasm/lib/scheduler/invoker_tasks.go b/chasm/lib/scheduler/invoker_tasks.go index 8dd7c554ffa..f32d66d7337 100644 --- a/chasm/lib/scheduler/invoker_tasks.go +++ b/chasm/lib/scheduler/invoker_tasks.go @@ -245,7 +245,8 @@ func (h *InvokerExecuteTaskHandler) Execute( // Invoker will never have work pending for more than one of these calls (terminate, // cancel, start) at a time, so it isn't sensible to run them in parallel. The // structure below is simply for code simplicity. - ictx := h.newInvokerTaskHandlerContext(ctx, scheduler) + // One budget is shared by all three phases, so pass the context by pointer. + ictx := new(h.newInvokerTaskHandlerContext(ctx, scheduler)) result = result.Append(h.terminateWorkflows(ictx, logger, metricsHandler, scheduler, invoker.GetTerminateWorkflows())) result = result.Append(h.cancelWorkflows(ictx, logger, metricsHandler, scheduler, invoker.GetCancelWorkflows())) result = result.Append(h.startWorkflows( @@ -300,7 +301,7 @@ func (i *invokerTaskHandlerContext) takeNextAction() bool { // cancelWorkflows does a best-effort attempt to cancel all workflow executions provided in targets. func (h *InvokerExecuteTaskHandler) cancelWorkflows( - ctx invokerTaskHandlerContext, + ctx *invokerTaskHandlerContext, logger log.Logger, metricsHandler metrics.Handler, scheduler *Scheduler, @@ -342,7 +343,7 @@ func (h *InvokerExecuteTaskHandler) cancelWorkflows( // terminateWorkflows does a best-effort attempt to terminate all workflow executions provided in targets. func (h *InvokerExecuteTaskHandler) terminateWorkflows( - ctx invokerTaskHandlerContext, + ctx *invokerTaskHandlerContext, logger log.Logger, metricsHandler metrics.Handler, scheduler *Scheduler, @@ -384,7 +385,7 @@ func (h *InvokerExecuteTaskHandler) terminateWorkflows( // startWorkflows executes the provided list of starts, returning a result with their outcomes. func (h *InvokerExecuteTaskHandler) startWorkflows( - ctx invokerTaskHandlerContext, + ctx *invokerTaskHandlerContext, logger log.Logger, metricsHandler metrics.Handler, scheduler *Scheduler, diff --git a/chasm/lib/scheduler/schedulertest/known_bugs_test.go b/chasm/lib/scheduler/schedulertest/known_bugs_test.go index e4e91069f22..1e6ffdcb434 100644 --- a/chasm/lib/scheduler/schedulertest/known_bugs_test.go +++ b/chasm/lib/scheduler/schedulertest/known_bugs_test.go @@ -13,16 +13,14 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" ) -// TestActionBudgetSharedAcrossPhases reproduces a bug found by -// TestSchedulerProperties. InvokerExecuteTask.Execute passes its +// TestActionBudgetSharedAcrossPhases pins a bug found by +// TestSchedulerProperties. InvokerExecuteTask.Execute used to pass its // invokerTaskHandlerContext by value to terminateWorkflows, cancelWorkflows and -// startWorkflows, so each phase spends from its own copy of the action budget. +// startWorkflows, so each phase spent from its own copy of the action budget. // A cancel from a CANCEL_OTHER start and a terminate from a later // TERMINATE_OTHER start both target the same running workflow, and one -// execution makes two RPCs although MaxActionsPerExecution is 1. +// execution made two RPCs although MaxActionsPerExecution was 1. func TestActionBudgetSharedAcrossPhases(t *testing.T) { - t.Skip("InvokerExecuteTask spends the MaxActionsPerExecution budget separately in the terminate, cancel and start phases (invoker_tasks.go Execute passes ictx by value)") - start := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) s := schedulertest.DefaultSchedule() s.Spec.Interval[0].Interval = durationpb.New(10 * time.Second) @@ -55,7 +53,7 @@ func TestActionBudgetSharedAcrossPhases(t *testing.T) { // keeps adding starts until the buffer holds about recentActionCount plus // MaxBufferSize/2 entries. func TestBackfillRespectsSmallMaxBufferSize(t *testing.T) { - t.Skip("backfillerBufferCapacity assumes recentActionCount completed starts are retained, so a backfill overfills a MaxBufferSize below 20 (backfiller_tasks.go allowedBufferedStarts)") + t.Skip("backfillerBufferCapacity discounts a constant recentActionCount (10) buffered starts as retained history, so a backfill overfills a MaxBufferSize below 20; upstream pins this arithmetic in TestAllowedBufferedStartsDiscountsRetainedHistory") start := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) s := schedulertest.DefaultSchedule() diff --git a/chasm/lib/scheduler/schedulertest/property_test.go b/chasm/lib/scheduler/schedulertest/property_test.go index 507a7dbffcf..20e12573124 100644 --- a/chasm/lib/scheduler/schedulertest/property_test.go +++ b/chasm/lib/scheduler/schedulertest/property_test.go @@ -46,27 +46,17 @@ func TestSchedulerProperties(t *testing.T) { // buffer (see TestBackfillRespectsSmallMaxBufferSize). maxBuffer := rapid.SampledFrom([]int{20, 24, 40}).Draw(rt, "maxBufferSize") reserve := rapid.IntRange(0, 1).Draw(rt, "generatorBufferReserve") - // CANCEL_OTHER and TERMINATE_OTHER put terminate and cancel RPCs in the same - // execution as starts, and the invoker spends the action budget separately - // per phase (see TestActionBudgetSharedAcrossPhases). Runs that use them - // therefore get a budget of 8 and no ALLOW_ALL, so at most one workflow runs - // and no execution reaches it; the budget invariant stays meaningful in the - // runs without them. 8 also keeps one execution's starts within the 10 - // recent actions the driver reads recorded starts from. + // The budget stays well below the 10 recent actions the driver reads + // recorded starts from, so one execution cannot trim them out of view. policies := []enumspb.ScheduleOverlapPolicy{ enumspb.SCHEDULE_OVERLAP_POLICY_SKIP, enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ONE, enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ALL, + enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL, + enumspb.SCHEDULE_OVERLAP_POLICY_CANCEL_OTHER, + enumspb.SCHEDULE_OVERLAP_POLICY_TERMINATE_OTHER, } maxActions := rapid.SampledFrom([]int{1, 2, 3}).Draw(rt, "maxActionsPerExecution") - if rapid.Bool().Draw(rt, "replacingPolicies") { - policies = append(policies, - enumspb.SCHEDULE_OVERLAP_POLICY_CANCEL_OTHER, - enumspb.SCHEDULE_OVERLAP_POLICY_TERMINATE_OTHER) - maxActions = 8 - } else { - policies = append(policies, enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL) - } sched := drawSchedule(rt, start, idle, policies) d := schedulertest.NewDriver(rt, From 2117d2824ed0ef6634c4eed584fa3b8f77e8b6b9 Mon Sep 17 00:00:00 2001 From: chaptersix <13949480+chaptersix@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:00:27 -0600 Subject: [PATCH 18/22] Keep scheduler tweakables at their defaults in the property test Stop drawing MaxBufferSize, the generator reserve and MaxActionsPerExecution; only idle time is varied. Drop the reproducer for backfill overfilling a MaxBufferSize below 20, which only occurs at a limit nobody configures, and rewrite the action-budget regression to fail at the default budget. Document the harness structure. Co-Authored-By: Claude Fable 5.2 --- .../schedulertest/action_budget_test.go | 51 ++++++++++++ .../schedulertest/known_bugs_test.go | 82 ------------------- .../scheduler/schedulertest/property_test.go | 14 ---- .../chasm-schedulertest-coverage-plan.md | 20 +++++ 4 files changed, 71 insertions(+), 96 deletions(-) create mode 100644 chasm/lib/scheduler/schedulertest/action_budget_test.go delete mode 100644 chasm/lib/scheduler/schedulertest/known_bugs_test.go diff --git a/chasm/lib/scheduler/schedulertest/action_budget_test.go b/chasm/lib/scheduler/schedulertest/action_budget_test.go new file mode 100644 index 00000000000..ecef1304dc1 --- /dev/null +++ b/chasm/lib/scheduler/schedulertest/action_budget_test.go @@ -0,0 +1,51 @@ +package schedulertest_test + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + enumspb "go.temporal.io/api/enums/v1" + schedulepb "go.temporal.io/api/schedule/v1" + "go.temporal.io/server/chasm/lib/scheduler" + "go.temporal.io/server/chasm/lib/scheduler/schedulertest" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// TestActionBudgetSharedAcrossPhases pins a bug found by +// TestSchedulerProperties. InvokerExecuteTask.Execute used to pass its +// invokerTaskHandlerContext by value to terminateWorkflows, cancelWorkflows and +// startWorkflows, so each phase spent from its own copy of the action budget. +// Here one execution has a cancel, a terminate and more ALLOW_ALL starts than +// the default MaxActionsPerExecution allows; together they must stay within it. +func TestActionBudgetSharedAcrossPhases(t *testing.T) { + start := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) + s := schedulertest.DefaultSchedule() + s.Policies.OverlapPolicy = enumspb.SCHEDULE_OVERLAP_POLICY_SKIP + s.State.Paused = true + var vs []schedulertest.Violation + d := schedulertest.NewDriver(t, + schedulertest.WithStartTime(start), + schedulertest.WithSchedule(s), + schedulertest.WithAfterStep(collectViolations(&vs)), + ) + backfill := func(policy enumspb.ScheduleOverlapPolicy, points int) { + from := start.Add(-time.Hour) + require.NoError(t, d.Backfill(&schedulepb.BackfillRequest{ + StartTime: timestamppb.New(from), + EndTime: timestamppb.New(from.Add(time.Duration(points-1) * schedulertest.DefaultInterval)), + OverlapPolicy: policy, + })) + } + + require.NoError(t, d.TriggerImmediately()) + require.True(t, d.StepSideEffect(), "start the triggered workflow") + backfill(enumspb.SCHEDULE_OVERLAP_POLICY_CANCEL_OTHER, 1) + backfill(enumspb.SCHEDULE_OVERLAP_POLICY_TERMINATE_OTHER, 1) + backfill(enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL, scheduler.DefaultTweakables.MaxActionsPerExecution) + for range 4 { + d.Step() + } + + require.Empty(t, violationNames(vs)) +} diff --git a/chasm/lib/scheduler/schedulertest/known_bugs_test.go b/chasm/lib/scheduler/schedulertest/known_bugs_test.go deleted file mode 100644 index 1e6ffdcb434..00000000000 --- a/chasm/lib/scheduler/schedulertest/known_bugs_test.go +++ /dev/null @@ -1,82 +0,0 @@ -package schedulertest_test - -import ( - "testing" - "time" - - "github.com/stretchr/testify/require" - enumspb "go.temporal.io/api/enums/v1" - schedulepb "go.temporal.io/api/schedule/v1" - "go.temporal.io/server/chasm/lib/scheduler" - "go.temporal.io/server/chasm/lib/scheduler/schedulertest" - "google.golang.org/protobuf/types/known/durationpb" - "google.golang.org/protobuf/types/known/timestamppb" -) - -// TestActionBudgetSharedAcrossPhases pins a bug found by -// TestSchedulerProperties. InvokerExecuteTask.Execute used to pass its -// invokerTaskHandlerContext by value to terminateWorkflows, cancelWorkflows and -// startWorkflows, so each phase spent from its own copy of the action budget. -// A cancel from a CANCEL_OTHER start and a terminate from a later -// TERMINATE_OTHER start both target the same running workflow, and one -// execution made two RPCs although MaxActionsPerExecution was 1. -func TestActionBudgetSharedAcrossPhases(t *testing.T) { - start := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) - s := schedulertest.DefaultSchedule() - s.Spec.Interval[0].Interval = durationpb.New(10 * time.Second) - s.Policies.OverlapPolicy = enumspb.SCHEDULE_OVERLAP_POLICY_SKIP - var vs []schedulertest.Violation - d := schedulertest.NewDriver(t, - schedulertest.WithStartTime(start), - schedulertest.WithSchedule(s), - schedulertest.WithConfig(configWith(func(tw *scheduler.Tweakables) { tw.MaxActionsPerExecution = 1 })), - schedulertest.WithAfterStep(collectViolations(&vs)), - ) - backfill := func(policy enumspb.ScheduleOverlapPolicy) { - at := timestamppb.New(start.Add(-time.Minute)) - require.NoError(t, d.Backfill(&schedulepb.BackfillRequest{StartTime: at, EndTime: at, OverlapPolicy: policy})) - } - - require.NoError(t, d.TriggerImmediately()) - backfill(enumspb.SCHEDULE_OVERLAP_POLICY_CANCEL_OTHER) - require.True(t, d.StepSideEffect(), "start the triggered workflow") - backfill(enumspb.SCHEDULE_OVERLAP_POLICY_TERMINATE_OTHER) - require.True(t, d.StepSideEffect(), "cancel and terminate the running workflow") - - require.Empty(t, violationNames(vs)) -} - -// TestBackfillRespectsSmallMaxBufferSize reproduces a bug found by -// TestSchedulerProperties. The backfiller's capacity calculation subtracts the -// constant recentActionCount (10) from the buffer length instead of the number -// of completed starts actually retained, so with a MaxBufferSize below 20 it -// keeps adding starts until the buffer holds about recentActionCount plus -// MaxBufferSize/2 entries. -func TestBackfillRespectsSmallMaxBufferSize(t *testing.T) { - t.Skip("backfillerBufferCapacity discounts a constant recentActionCount (10) buffered starts as retained history, so a backfill overfills a MaxBufferSize below 20; upstream pins this arithmetic in TestAllowedBufferedStartsDiscountsRetainedHistory") - - start := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) - s := schedulertest.DefaultSchedule() - s.Policies.OverlapPolicy = enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ALL - s.State.Paused = true - var vs []schedulertest.Violation - d := schedulertest.NewDriver(t, - schedulertest.WithStartTime(start.Add(10*time.Minute)), - schedulertest.WithSchedule(s), - schedulertest.WithConfig(configWith(func(tw *scheduler.Tweakables) { - tw.MaxBufferSize = 2 - tw.GeneratorBufferReserveSize = 0 - })), - schedulertest.WithAfterStep(collectViolations(&vs)), - ) - require.NoError(t, d.Backfill(&schedulepb.BackfillRequest{ - StartTime: timestamppb.New(start), - EndTime: timestamppb.New(start.Add(10 * time.Minute)), - OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ALL, - })) - for range 12 { - d.Step() - } - - require.Empty(t, violationNames(vs)) -} diff --git a/chasm/lib/scheduler/schedulertest/property_test.go b/chasm/lib/scheduler/schedulertest/property_test.go index 20e12573124..94a289827ac 100644 --- a/chasm/lib/scheduler/schedulertest/property_test.go +++ b/chasm/lib/scheduler/schedulertest/property_test.go @@ -38,16 +38,6 @@ func TestSchedulerProperties(t *testing.T) { // A small IdleTime lets fire-time gaps cross the idle window, which the // 7d default never does with these intervals. idle := rapid.SampledFrom([]time.Duration{10 * time.Minute, time.Hour, 2 * time.Hour}).Draw(rt, "idleTime") - // Small buffer and action limits so the buffer-bound and action-budget - // invariants are reachable; the defaults (1000 and 5) are not. A reserve - // below MaxBufferSize/2 leaves the backfiller room to make progress. - // - // MaxBufferSize stays at or above 20: the backfiller overfills a smaller - // buffer (see TestBackfillRespectsSmallMaxBufferSize). - maxBuffer := rapid.SampledFrom([]int{20, 24, 40}).Draw(rt, "maxBufferSize") - reserve := rapid.IntRange(0, 1).Draw(rt, "generatorBufferReserve") - // The budget stays well below the 10 recent actions the driver reads - // recorded starts from, so one execution cannot trim them out of view. policies := []enumspb.ScheduleOverlapPolicy{ enumspb.SCHEDULE_OVERLAP_POLICY_SKIP, enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ONE, @@ -56,7 +46,6 @@ func TestSchedulerProperties(t *testing.T) { enumspb.SCHEDULE_OVERLAP_POLICY_CANCEL_OTHER, enumspb.SCHEDULE_OVERLAP_POLICY_TERMINATE_OTHER, } - maxActions := rapid.SampledFrom([]int{1, 2, 3}).Draw(rt, "maxActionsPerExecution") sched := drawSchedule(rt, start, idle, policies) d := schedulertest.NewDriver(rt, @@ -64,9 +53,6 @@ func TestSchedulerProperties(t *testing.T) { schedulertest.WithSchedule(sched), schedulertest.WithConfig(configWith(func(tw *scheduler.Tweakables) { tw.IdleTime = idle - tw.MaxBufferSize = maxBuffer - tw.GeneratorBufferReserveSize = reserve - tw.MaxActionsPerExecution = maxActions })), // InvokerExecuteTask may lag the timers, and workflows may run, but // neither for a whole idle window. Past that, V1 and V2 both close diff --git a/docs/architecture/chasm-schedulertest-coverage-plan.md b/docs/architecture/chasm-schedulertest-coverage-plan.md index b9d6346a55b..b221b5e4247 100644 --- a/docs/architecture/chasm-schedulertest-coverage-plan.md +++ b/docs/architecture/chasm-schedulertest-coverage-plan.md @@ -115,6 +115,26 @@ Each fix has a default-run regression; none is gated. CANCEL_OTHER still act. `TestSelectedStartReservesOverlapSlot`, `TestFailedSelectedStartReleasesOverlapSlot`, `TestSelectedStartReservationStillTerminatesOther`. +- **Action budget shared across phases.** `InvokerExecuteTask` passed its + per-task context by value, so the terminate, cancel and start phases each + spent `MaxActionsPerExecution` separately. It is now passed by pointer. Found + by the widened property. `TestActionBudgetSharedAcrossPhases`. + +## Harness structure + +The component-agnostic stepping loop is `chasmtest.TaskRunner`: it finds the +next pure or side-effect task, validates and dispatches it, advances the clock, +and exposes hooks for component-specific behaviour. `schedulertest` supplies the +scheduler's mocks, operations, `Snapshot` and invariants on top of it. The +engine, the runner and the driver take `chasmtest.TB`, which `*rapid.T` +satisfies, so a harness failure inside a property shrinks like any other. + +`TestSchedulerProperties` runs as a Rapid state machine (`rt.Repeat`) over +stepping, completion, pause, unpause, trigger, backfill, update, failed starts +and lost start responses, across all six overlap policies. Invariants run in the +after-step hook, so they also see the settled states inside a compound action. +Idle time is drawn; the other tweakables stay at their defaults. Rollback +migration is not a property operation. ## Unresolved counterexamples and limits From 471e17bafe54e53b28b038e82336c5b93f7f2c4c Mon Sep 17 00:00:00 2001 From: chaptersix <13949480+chaptersix@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:25:04 -0600 Subject: [PATCH 19/22] Share scheduler test fixtures between the unit tests and the lifecycle harness schedulertest is now the one home of the default schedule and config, the spec processor, the library wiring and a SchedulerEngine wrapper with typed read and update helpers. helper_test.go keeps its existing names as thin wrappers over them, and the Driver builds its engine through NewEngine. Co-Authored-By: Claude Fable 5.2 --- chasm/lib/scheduler/helper_test.go | 198 +++++------------ chasm/lib/scheduler/schedulertest/driver.go | 21 +- chasm/lib/scheduler/schedulertest/engine.go | 221 +++++++++++++++++++ chasm/lib/scheduler/schedulertest/library.go | 120 ++++++---- 4 files changed, 363 insertions(+), 197 deletions(-) create mode 100644 chasm/lib/scheduler/schedulertest/engine.go diff --git a/chasm/lib/scheduler/helper_test.go b/chasm/lib/scheduler/helper_test.go index 295a431d761..303aef2c960 100644 --- a/chasm/lib/scheduler/helper_test.go +++ b/chasm/lib/scheduler/helper_test.go @@ -7,15 +7,14 @@ import ( "time" "github.com/stretchr/testify/require" - commonpb "go.temporal.io/api/common/v1" schedulepb "go.temporal.io/api/schedule/v1" - workflowpb "go.temporal.io/api/workflow/v1" "go.temporal.io/api/workflowservice/v1" persistencespb "go.temporal.io/server/api/persistence/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/chasmtest" "go.temporal.io/server/chasm/lib/scheduler" "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" + "go.temporal.io/server/chasm/lib/scheduler/schedulertest" "go.temporal.io/server/common/backoff" "go.temporal.io/server/common/clock" "go.temporal.io/server/common/log" @@ -25,115 +24,48 @@ import ( "go.temporal.io/server/service/history/tasks" legacyscheduler "go.temporal.io/server/service/worker/scheduler" "go.uber.org/mock/gomock" - "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/timestamppb" ) const ( - namespace = "ns" - namespaceID = "ns-id" - scheduleID = "sched-id" + namespace = schedulertest.Namespace + namespaceID = schedulertest.NamespaceID + scheduleID = schedulertest.ScheduleID - defaultInterval = 1 * time.Minute - defaultCatchupWindow = 5 * time.Minute + defaultInterval = schedulertest.DefaultInterval + defaultCatchupWindow = schedulertest.DefaultCatchupWindow ) // newLegacySpecBuilder builds a legacy SpecBuilder with the given warn/max compute-limit bounds. // A value of 0 means "use the default" (GetNextTime treats a non-positive bound as its default). func newLegacySpecBuilder(warnIter, maxIter int) *legacyscheduler.SpecBuilder { - return legacyscheduler.NewSpecBuilder(func() int { return warnIter }, func() int { return maxIter }) + return schedulertest.NewSpecBuilder(warnIter, maxIter) } // defaultSchedule returns a protobuf definition for a schedule matching this // package's other testing defaults. func defaultSchedule() *schedulepb.Schedule { - return &schedulepb.Schedule{ - Spec: &schedulepb.ScheduleSpec{ - Interval: []*schedulepb.IntervalSpec{ - { - Interval: durationpb.New(defaultInterval), - Phase: durationpb.New(0), - }, - }, - }, - Action: &schedulepb.ScheduleAction{ - Action: &schedulepb.ScheduleAction_StartWorkflow{ - StartWorkflow: &workflowpb.NewWorkflowExecutionInfo{ - WorkflowId: "scheduled-wf", - WorkflowType: &commonpb.WorkflowType{Name: "scheduled-wf-type"}, - }, - }, - }, - Policies: &schedulepb.SchedulePolicies{ - CatchupWindow: durationpb.New(defaultCatchupWindow), - }, - State: &schedulepb.ScheduleState{ - Paused: false, - LimitedActions: false, - RemainingActions: 0, - }, - } + return schedulertest.DefaultSchedule() } +// defaultConfig is the shared default config with the production-style jittered +// exponential retry policy. The lifecycle harness swaps in a deterministic one +// so that rapid replays a failure identically; these tests never needed that. func defaultConfig() *scheduler.Config { - return &scheduler.Config{ - Tweakables: func(_ string) scheduler.Tweakables { - return scheduler.DefaultTweakables - }, - ServiceCallTimeout: func() time.Duration { - return 5 * time.Second - }, - EncodeInternalTokenWithEnvelope: func(string) bool { - return true - }, - RetryPolicy: func() backoff.RetryPolicy { - return backoff.NewExponentialRetryPolicy(1 * time.Second) - }, + cfg := schedulertest.DefaultConfig() + cfg.RetryPolicy = func() backoff.RetryPolicy { + return backoff.NewExponentialRetryPolicy(1 * time.Second) } + return cfg } func newTestLibrary(logger log.Logger, specProcessor scheduler.SpecProcessor) *scheduler.Library { - config := defaultConfig() - specBuilder := newLegacySpecBuilder(0, 0) - invokerOpts := scheduler.InvokerTaskHandlerOptions{ - Config: config, - MetricsHandler: metrics.NoopMetricsHandler, - BaseLogger: logger, - SpecProcessor: specProcessor, - } - return scheduler.NewLibrary( - config, - nil, - scheduler.NewSchedulerIdleTaskHandler(scheduler.SchedulerIdleTaskHandlerOptions{ - Config: config, - MetricsHandler: metrics.NoopMetricsHandler, - BaseLogger: logger, - }), - scheduler.NewSchedulerCallbacksTaskHandler(scheduler.SchedulerCallbacksTaskHandlerOptions{ - Config: config, - MetricsHandler: metrics.NoopMetricsHandler, - }), - scheduler.NewGeneratorTaskHandler(scheduler.GeneratorTaskHandlerOptions{ - Config: config, - MetricsHandler: metrics.NoopMetricsHandler, - BaseLogger: logger, - SpecProcessor: specProcessor, - SpecBuilder: specBuilder, - }), - scheduler.NewInvokerExecuteTaskHandler(invokerOpts), - scheduler.NewInvokerProcessBufferTaskHandler(invokerOpts), - scheduler.NewBackfillerTaskHandler(scheduler.BackfillerTaskHandlerOptions{ - Config: config, - MetricsHandler: metrics.NoopMetricsHandler, - BaseLogger: logger, - SpecProcessor: specProcessor, - }), - scheduler.NewSchedulerMigrateToWorkflowTaskHandler(scheduler.SchedulerMigrateToWorkflowTaskHandlerOptions{ - Config: config, - MetricsHandler: metrics.NoopMetricsHandler, - BaseLogger: logger, - }), - ) + return schedulertest.NewTestLibrary(schedulertest.LibraryOptions{ + Config: defaultConfig(), + Logger: logger, + SpecProcessor: specProcessor, + SpecBuilder: newLegacySpecBuilder(0, 0), + }) } // testEnv holds all components needed for scheduler tests. @@ -201,17 +133,7 @@ func expiredSchedule(now time.Time) *schedulepb.Schedule { // newRealSpecProcessor creates a real SpecProcessor for tests. func newRealSpecProcessor(ctrl *gomock.Controller, logger log.Logger) scheduler.SpecProcessor { - mockMetrics := metrics.NewMockHandler(ctrl) - mockMetrics.EXPECT().Counter(gomock.Any()).Return(metrics.NoopCounterMetricFunc).AnyTimes() - mockMetrics.EXPECT().WithTags(gomock.Any()).Return(mockMetrics).AnyTimes() - mockMetrics.EXPECT().Timer(gomock.Any()).Return(metrics.NoopTimerMetricFunc).AnyTimes() - - return scheduler.NewSpecProcessor( - defaultConfig(), - mockMetrics, - logger, - newLegacySpecBuilder(0, 0), - ) + return schedulertest.NewSpecProcessor(ctrl, logger, newLegacySpecBuilder(0, 0)) } // engineTestConfig holds configuration options for newTestEngineContext. @@ -237,7 +159,6 @@ func withEngineSpecProcessor(sp scheduler.SpecProcessor) engineTestOption { func withEngineTimeSource(ts *clock.EventTimeSource) engineTestOption { return func(c *engineTestConfig) { c.timeSource = ts - c.engineOpts = append(c.engineOpts, chasmtest.WithTimeSource(ts)) } } @@ -268,28 +189,37 @@ func newTestEngineContextFromConfig( logger log.Logger, config *engineTestConfig, ) (*chasmtest.Engine, context.Context) { - specProcessor := config.specProcessor - if specProcessor == nil { - specProcessor = newRealSpecProcessor(gomock.NewController(t), logger) + base := newBaseEngine(t, logger, config) + return base.Engine, base.Ctx +} + +// newBaseEngine builds the shared scheduler engine with this package's config, +// spec builder and stuckness invariants. +func newBaseEngine(t *testing.T, logger log.Logger, config *engineTestConfig) *schedulertest.SchedulerEngine { + opts := []schedulertest.Option{ + schedulertest.WithLogger(logger), + schedulertest.WithSchedulerConfig(defaultConfig()), + schedulertest.WithSpecBuilder(newLegacySpecBuilder(0, 0)), + schedulertest.WithEngineOptions(config.engineOpts...), + schedulertest.WithSchedulerInvariant(func(t chasmtest.TB, registry *chasm.Registry, node *chasm.Node, sched *scheduler.Scheduler) { + requireNotStuck(t, node, sched) + requireIdleCloseTimeBacked(t, registry, node, sched) + }), } - - registry := chasm.NewRegistry(logger) - require.NoError(t, registry.Register(&chasm.CoreLibrary{})) - require.NoError(t, registry.Register(newTestLibrary(logger, specProcessor))) - - config.engineOpts = append(config.engineOpts, chasmtest.WithInvariantCheck( - func(t chasmtest.TB, node *chasm.Node, root chasm.RootComponent) { - requireValidSchedulerState(t, registry, node, root) - }, - )) - engine := chasmtest.NewEngine(t, registry, config.engineOpts...) - return engine, chasm.NewEngineContext(context.Background(), engine) + if config.specProcessor != nil { + opts = append(opts, schedulertest.WithSpecProcessor(config.specProcessor)) + } + if config.timeSource != nil { + opts = append(opts, schedulertest.WithTimeSource(config.timeSource)) + } + return schedulertest.NewEngine(t, opts...) } // schedulerTestEngine bundles a CHASM test engine, a created Scheduler root // component, and convenience accessors, for tests that drive a Scheduler // through its actual task handlers rather than the newTestEnv rapid harness. type schedulerTestEngine struct { + base *schedulertest.SchedulerEngine engine *chasmtest.Engine engineCtx context.Context rootRef chasm.ComponentRef @@ -313,14 +243,9 @@ func newSchedulerTestEngine( if config.timeSource == nil { config.timeSource = clock.NewEventTimeSource() config.timeSource.Update(time.Now()) - config.engineOpts = append(config.engineOpts, chasmtest.WithTimeSource(config.timeSource)) } - engine, engineCtx := newTestEngineContextFromConfig(t, logger, config) - rootRef := chasm.NewComponentRef[*scheduler.Scheduler](chasm.ExecutionKey{ - NamespaceID: namespaceID, - BusinessID: scheduleID, - }) - _, err := scheduler.NewTestHandler(logger).CreateSchedule(engineCtx, &schedulerpb.CreateScheduleRequest{ + base := newBaseEngine(t, logger, config) + _, err := scheduler.NewTestHandler(logger).CreateSchedule(base.Ctx, &schedulerpb.CreateScheduleRequest{ NamespaceId: namespaceID, FrontendRequest: &workflowservice.CreateScheduleRequest{ Namespace: namespace, @@ -331,9 +256,10 @@ func newSchedulerTestEngine( }) require.NoError(t, err) return &schedulerTestEngine{ - engine: engine, - engineCtx: engineCtx, - rootRef: rootRef, + base: base, + engine: base.Engine, + engineCtx: base.Ctx, + rootRef: base.Ref, logger: logger, timeSource: config.timeSource, } @@ -344,15 +270,7 @@ func newSchedulerTestEngine( func (e *schedulerTestEngine) updateScheduler( update func(*scheduler.Scheduler, chasm.MutableContext) error, ) error { - _, _, err := chasm.UpdateComponent( - e.engineCtx, - e.rootRef, - func(s *scheduler.Scheduler, ctx chasm.MutableContext, _ struct{}) (struct{}, error) { - return struct{}{}, update(s, ctx) - }, - struct{}{}, - ) - return err + return e.base.Update(update) } // readScheduler runs read against the Scheduler root component through the @@ -360,15 +278,7 @@ func (e *schedulerTestEngine) updateScheduler( func (e *schedulerTestEngine) readScheduler( read func(*scheduler.Scheduler, chasm.Context) error, ) error { - _, err := chasm.ReadComponent( - e.engineCtx, - e.rootRef, - func(s *scheduler.Scheduler, ctx chasm.Context, _ struct{}) (struct{}, error) { - return struct{}{}, read(s, ctx) - }, - struct{}{}, - ) - return err + return e.base.Read(read) } // newTestEnv creates a new test environment with the given options. diff --git a/chasm/lib/scheduler/schedulertest/driver.go b/chasm/lib/scheduler/schedulertest/driver.go index de5ef3f163a..5d902753b5c 100644 --- a/chasm/lib/scheduler/schedulertest/driver.go +++ b/chasm/lib/scheduler/schedulertest/driver.go @@ -295,19 +295,18 @@ func newDriverScaffold(t chasmtest.TB, opts ...DriverOption) (*Driver, *driverCo }, nil }).AnyTimes() - specProcessor := NewRealSpecProcessor(ctrl, logger) - - registry := chasm.NewRegistry(logger) - if err := registry.Register(&chasm.CoreLibrary{}); err != nil { - t.Fatalf("failed to register core library: %v", err) - } metricsHandler := metricstest.NewCaptureHandler() capture := metricsHandler.StartCapture() - if err := registry.Register(NewTestLibrary(cfg.config, logger, metricsHandler, ts, specProcessor, frontend, history)); err != nil { - t.Fatalf("failed to register scheduler library: %v", err) - } - - engine := chasmtest.NewEngine(t, registry, chasmtest.WithTimeSource(ts)) + base := NewEngine(t, + WithLogger(logger), + WithSchedulerConfig(cfg.config), + WithTimeSource(ts), + WithInvokerTimeSource(ts), + WithInvokerMetrics(metricsHandler), + WithClients(frontend, history), + ) + engine := base.Engine + registry := base.Registry d := &Driver{ recordCalls: make(chan *RecordCall), diff --git a/chasm/lib/scheduler/schedulertest/engine.go b/chasm/lib/scheduler/schedulertest/engine.go new file mode 100644 index 00000000000..ac0f9373869 --- /dev/null +++ b/chasm/lib/scheduler/schedulertest/engine.go @@ -0,0 +1,221 @@ +package schedulertest + +import ( + "context" + + "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/chasmtest" + "go.temporal.io/server/chasm/lib/scheduler" + "go.temporal.io/server/common/clock" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/resource" + "go.temporal.io/server/common/testing/testlogger" + legacyscheduler "go.temporal.io/server/service/worker/scheduler" + "go.uber.org/mock/gomock" +) + +// CreateRequestID is the request ID [SchedulerEngine.Start] records for the +// scheduler's creating transaction. +const CreateRequestID = "create-req" + +// SchedulerEngine bundles a [chasmtest.Engine] with the CHASM registry that has +// the scheduler library registered, the reference of the Scheduler root +// component, and typed read/update helpers. It is the shared base of the unit +// tests in package scheduler_test and of [Driver]. +// +// NewEngine only builds the engine. Create the execution with [SchedulerEngine.Start], +// or through a handler using [SchedulerEngine.Ctx]. +type SchedulerEngine struct { + T chasmtest.TB + Engine *chasmtest.Engine + Registry *chasm.Registry + Logger log.Logger + // Config is the scheduler config the library was wired with. + Config *scheduler.Config + // TimeSource is the clock the engine was built with, or nil if the engine + // kept its default. + TimeSource *clock.EventTimeSource + // SpecProcessor is the processor the library was wired with. + SpecProcessor scheduler.SpecProcessor + // Ctx carries the engine, ready for chasm.StartExecution, chasm.ReadComponent + // and handler calls. + Ctx context.Context + Key chasm.ExecutionKey + Ref chasm.ComponentRef +} + +type engineOptions struct { + logger log.Logger + library LibraryOptions + engineOpts []chasmtest.EngineOption + timeSource *clock.EventTimeSource + invariants []func(chasmtest.TB, *chasm.Registry, *chasm.Node, *scheduler.Scheduler) +} + +// Option configures [NewEngine]. +type Option func(*engineOptions) + +// WithLogger overrides the default test logger. +func WithLogger(l log.Logger) Option { + return func(o *engineOptions) { o.logger = l } +} + +// WithSchedulerConfig overrides the scheduler config (default: [DefaultConfig]). Use it +// to set small Tweakables, or a different retry policy. +func WithSchedulerConfig(cfg *scheduler.Config) Option { + return func(o *engineOptions) { o.library.Config = cfg } +} + +// WithSpecProcessor overrides the real spec processor, e.g. to simulate failures. +func WithSpecProcessor(sp scheduler.SpecProcessor) Option { + return func(o *engineOptions) { o.library.SpecProcessor = sp } +} + +// WithSpecBuilder overrides the spec builder used by both the library and the +// default spec processor (default: [DefaultSpecBuilder]). +func WithSpecBuilder(b *legacyscheduler.SpecBuilder) Option { + return func(o *engineOptions) { o.library.SpecBuilder = b } +} + +// WithTimeSource gives the engine a controllable clock. Without it the engine +// keeps chasmtest's default, which is frozen at time.Now. +func WithTimeSource(ts *clock.EventTimeSource) Option { + return func(o *engineOptions) { + o.timeSource = ts + o.engineOpts = append(o.engineOpts, chasmtest.WithTimeSource(ts)) + } +} + +// WithInvokerTimeSource also hands the clock to the invoker, which stamps +// StartTime on started workflows. Without it the invoker uses real time. +func WithInvokerTimeSource(ts clock.TimeSource) Option { + return func(o *engineOptions) { o.library.TimeSource = ts } +} + +// WithInvokerMetrics gives the invoker task handlers a metrics handler. +func WithInvokerMetrics(h metrics.Handler) Option { + return func(o *engineOptions) { o.library.MetricsHandler = h } +} + +// WithClients wires the frontend and history clients behind the side-effect +// task handlers (execute, callbacks, migrate). +func WithClients(frontend workflowservice.WorkflowServiceClient, history resource.HistoryClient) Option { + return func(o *engineOptions) { + o.library.FrontendClient = frontend + o.library.HistoryClient = history + } +} + +// WithEngineOptions passes options straight to [chasmtest.NewEngine], e.g. an +// extra invariant check or an engine metrics handler. +func WithEngineOptions(opts ...chasmtest.EngineOption) Option { + return func(o *engineOptions) { o.engineOpts = append(o.engineOpts, opts...) } +} + +// WithSchedulerInvariant installs a check run after every successful transition +// with the Scheduler root. It is skipped for roots that are not a Scheduler. +func WithSchedulerInvariant(check func(chasmtest.TB, *chasm.Registry, *chasm.Node, *scheduler.Scheduler)) Option { + return func(o *engineOptions) { o.invariants = append(o.invariants, check) } +} + +// NewEngine builds a registry with the core and scheduler libraries, wraps it in +// a [chasmtest.Engine], and returns the wrapper. No execution is started. +func NewEngine(t chasmtest.TB, opts ...Option) *SchedulerEngine { + t.Helper() + + o := &engineOptions{} + for _, opt := range opts { + opt(o) + } + if o.logger == nil { + o.logger = testlogger.NewTestLogger(t, testlogger.FailOnExpectedErrorOnly) + } + if o.library.Config == nil { + o.library.Config = DefaultConfig() + } + if o.library.SpecBuilder == nil { + o.library.SpecBuilder = DefaultSpecBuilder() + } + if o.library.SpecProcessor == nil { + o.library.SpecProcessor = NewSpecProcessor(gomock.NewController(t), o.logger, o.library.SpecBuilder) + } + o.library.Logger = o.logger + + registry := chasm.NewRegistry(o.logger) + if err := registry.Register(&chasm.CoreLibrary{}); err != nil { + t.Fatalf("failed to register core library: %v", err) + } + if err := registry.Register(NewTestLibrary(o.library)); err != nil { + t.Fatalf("failed to register scheduler library: %v", err) + } + + engineOpts := o.engineOpts + for _, check := range o.invariants { + engineOpts = append(engineOpts, chasmtest.WithInvariantCheck( + func(t chasmtest.TB, node *chasm.Node, root chasm.RootComponent) { + t.Helper() + sched, ok := root.(*scheduler.Scheduler) + if !ok { + t.Fatalf("scheduler test engine root has unexpected type %T", root) + } + check(t, registry, node, sched) + }, + )) + } + engine := chasmtest.NewEngine(t, registry, engineOpts...) + + key := chasm.ExecutionKey{NamespaceID: NamespaceID, BusinessID: ScheduleID} + return &SchedulerEngine{ + T: t, + Engine: engine, + Registry: registry, + Logger: o.logger, + Config: o.library.Config, + TimeSource: o.timeSource, + SpecProcessor: o.library.SpecProcessor, + Ctx: chasm.NewEngineContext(context.Background(), engine), + Key: key, + Ref: chasm.NewComponentRef[*scheduler.Scheduler](key), + } +} + +// Start starts the scheduler execution with startFn, recording [CreateRequestID]. +func (e *SchedulerEngine) Start(startFn func(chasm.MutableContext) (*scheduler.Scheduler, error)) error { + e.T.Helper() + _, err := chasm.StartExecution(e.Ctx, e.Key, func(mctx chasm.MutableContext, _ *struct{}) (*scheduler.Scheduler, error) { + return startFn(mctx) + }, (*struct{})(nil), chasm.WithRequestID(CreateRequestID)) + return err +} + +// Update runs update against the Scheduler root component through the engine's +// UpdateComponent path. +func (e *SchedulerEngine) Update(update func(*scheduler.Scheduler, chasm.MutableContext) error) error { + e.T.Helper() + _, _, err := chasm.UpdateComponent( + e.Ctx, + e.Ref, + func(s *scheduler.Scheduler, ctx chasm.MutableContext, _ struct{}) (struct{}, error) { + return struct{}{}, update(s, ctx) + }, + struct{}{}, + ) + return err +} + +// Read runs read against the Scheduler root component through the engine's +// read-only ReadComponent path. +func (e *SchedulerEngine) Read(read func(*scheduler.Scheduler, chasm.Context) error) error { + e.T.Helper() + _, err := chasm.ReadComponent( + e.Ctx, + e.Ref, + func(s *scheduler.Scheduler, ctx chasm.Context, _ struct{}) (struct{}, error) { + return struct{}{}, read(s, ctx) + }, + struct{}{}, + ) + return err +} diff --git a/chasm/lib/scheduler/schedulertest/library.go b/chasm/lib/scheduler/schedulertest/library.go index 8cbbdd18758..3a1432a2adf 100644 --- a/chasm/lib/scheduler/schedulertest/library.go +++ b/chasm/lib/scheduler/schedulertest/library.go @@ -6,9 +6,11 @@ // stuck without a task to drive the next state"). // // It deliberately lives in a non-test package so that property tests -// (pgregory.net/rapid) and the V1<->V2 migration round-trip test can import the -// driver. The helpers below mirror chasm/lib/scheduler/helper_test.go, but are -// exported and accept the mock clients the side-effect task handlers need. +// (pgregory.net/rapid), the V1<->V2 migration round-trip test and the unit tests +// in package scheduler_test can all import it. It is the single home of the +// scheduler fixtures (default schedule and config, spec processor, library +// wiring) and of [SchedulerEngine], the typed wrapper around a chasmtest engine +// that helper_test.go and [Driver] are both built on. package schedulertest import ( @@ -98,51 +100,84 @@ func (deterministicRetryPolicy) ComputeNextDelay(_ time.Duration, numAttempts in var _ backoff.RetryPolicy = deterministicRetryPolicy{} -// NewRealSpecProcessor builds a real SpecProcessor with no-op metrics, matching -// production spec processing so generated buffered starts and next-times are -// realistic. -func NewRealSpecProcessor(ctrl *gomock.Controller, logger log.Logger) scheduler.SpecProcessor { +// NewSpecBuilder returns a legacy SpecBuilder with the given warn/max compute-limit +// bounds. A value of 0 means "use the default": GetNextTime treats a non-positive +// warn bound as its default and a non-positive max bound as unlimited. +func NewSpecBuilder(warnIter, maxIter int) *legacyscheduler.SpecBuilder { + return legacyscheduler.NewSpecBuilder(func() int { return warnIter }, func() int { return maxIter }) +} + +// DefaultSpecBuilder returns a SpecBuilder bounded by the production dynamic +// config defaults. +func DefaultSpecBuilder() *legacyscheduler.SpecBuilder { + return legacyscheduler.NewSpecBuilder( + dynamicconfig.SchedulerSpecWarnIterations.Get(dynamicconfig.NewNoopCollection()), + dynamicconfig.SchedulerSpecMaxIterations.Get(dynamicconfig.NewNoopCollection()), + ) +} + +// NewSpecProcessor builds a real SpecProcessor over builder with no-op metrics, +// matching production spec processing so generated buffered starts and +// next-times are realistic. +func NewSpecProcessor(ctrl *gomock.Controller, logger log.Logger, builder *legacyscheduler.SpecBuilder) scheduler.SpecProcessor { mockMetrics := metrics.NewMockHandler(ctrl) mockMetrics.EXPECT().Counter(gomock.Any()).Return(metrics.NoopCounterMetricFunc).AnyTimes() mockMetrics.EXPECT().WithTags(gomock.Any()).Return(mockMetrics).AnyTimes() mockMetrics.EXPECT().Timer(gomock.Any()).Return(metrics.NoopTimerMetricFunc).AnyTimes() - return scheduler.NewSpecProcessor( - DefaultConfig(), - mockMetrics, - logger, - legacyscheduler.NewSpecBuilder( - dynamicconfig.SchedulerSpecWarnIterations.Get(dynamicconfig.NewNoopCollection()), - dynamicconfig.SchedulerSpecMaxIterations.Get(dynamicconfig.NewNoopCollection()), - ), - ) + return scheduler.NewSpecProcessor(DefaultConfig(), mockMetrics, logger, builder) +} + +// NewRealSpecProcessor is NewSpecProcessor over [DefaultSpecBuilder]. +func NewRealSpecProcessor(ctrl *gomock.Controller, logger log.Logger) scheduler.SpecProcessor { + return NewSpecProcessor(ctrl, logger, DefaultSpecBuilder()) +} + +// LibraryOptions wires a scheduler Library. Zero values select the defaults +// noted on each field. +type LibraryOptions struct { + // Config defaults to [DefaultConfig]. + Config *scheduler.Config + Logger log.Logger + // MetricsHandler is given to the invoker task handlers; the other handlers + // always use the no-op handler. Defaults to the no-op handler. + MetricsHandler metrics.Handler + // TimeSource stamps StartTime in the invoker. Nil leaves the handler's + // default (real time). + TimeSource clock.TimeSource + SpecProcessor scheduler.SpecProcessor + // SpecBuilder defaults to [DefaultSpecBuilder]. + SpecBuilder *legacyscheduler.SpecBuilder + // FrontendClient and HistoryClient back the side-effect task handlers + // (execute/callbacks/migrate). Nil unless the test exercises those paths. + FrontendClient workflowservice.WorkflowServiceClient + HistoryClient resource.HistoryClient } // NewTestLibrary builds a scheduler Library with all task handlers wired to the -// supplied spec processor and mock clients. The clients back the side-effect -// task handlers (execute/callbacks/migrate); pass mocks whose StartWorkflow / -// Terminate / Cancel expectations the test controls. -func NewTestLibrary( - config *scheduler.Config, - logger log.Logger, - metricsHandler metrics.Handler, - timeSource clock.TimeSource, - specProcessor scheduler.SpecProcessor, - frontendClient workflowservice.WorkflowServiceClient, - historyClient resource.HistoryClient, -) *scheduler.Library { - specBuilder := legacyscheduler.NewSpecBuilder( - dynamicconfig.SchedulerSpecWarnIterations.Get(dynamicconfig.NewNoopCollection()), - dynamicconfig.SchedulerSpecMaxIterations.Get(dynamicconfig.NewNoopCollection()), - ) +// supplied options. +func NewTestLibrary(o LibraryOptions) *scheduler.Library { + config := o.Config + if config == nil { + config = DefaultConfig() + } + invokerMetrics := o.MetricsHandler + if invokerMetrics == nil { + invokerMetrics = metrics.NoopMetricsHandler + } + specBuilder := o.SpecBuilder + if specBuilder == nil { + specBuilder = DefaultSpecBuilder() + } + logger := o.Logger invokerOpts := scheduler.InvokerTaskHandlerOptions{ Config: config, - MetricsHandler: metricsHandler, + MetricsHandler: invokerMetrics, BaseLogger: logger, - SpecProcessor: specProcessor, - HistoryClient: historyClient, - FrontendClient: frontendClient, - TimeSource: timeSource, + SpecProcessor: o.SpecProcessor, + HistoryClient: o.HistoryClient, + FrontendClient: o.FrontendClient, + TimeSource: o.TimeSource, } return scheduler.NewLibrary( config, @@ -154,14 +189,15 @@ func NewTestLibrary( }), scheduler.NewSchedulerCallbacksTaskHandler(scheduler.SchedulerCallbacksTaskHandlerOptions{ Config: config, - HistoryClient: historyClient, - FrontendClient: frontendClient, + MetricsHandler: metrics.NoopMetricsHandler, + HistoryClient: o.HistoryClient, + FrontendClient: o.FrontendClient, }), scheduler.NewGeneratorTaskHandler(scheduler.GeneratorTaskHandlerOptions{ Config: config, MetricsHandler: metrics.NoopMetricsHandler, BaseLogger: logger, - SpecProcessor: specProcessor, + SpecProcessor: o.SpecProcessor, SpecBuilder: specBuilder, }), scheduler.NewInvokerExecuteTaskHandler(invokerOpts), @@ -170,13 +206,13 @@ func NewTestLibrary( Config: config, MetricsHandler: metrics.NoopMetricsHandler, BaseLogger: logger, - SpecProcessor: specProcessor, + SpecProcessor: o.SpecProcessor, }), scheduler.NewSchedulerMigrateToWorkflowTaskHandler(scheduler.SchedulerMigrateToWorkflowTaskHandlerOptions{ Config: config, MetricsHandler: metrics.NoopMetricsHandler, BaseLogger: logger, - HistoryClient: historyClient, + HistoryClient: o.HistoryClient, SaMapperProvider: searchattribute.NewTestMapperProvider(nil), }), ) From 2bda549db3a3967421bd9ad776183262a741f1ec Mon Sep 17 00:00:00 2001 From: chaptersix <13949480+chaptersix@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:34:41 -0600 Subject: [PATCH 20/22] Check the stuck and idle-deadline invariants from one place The stuck and unbacked-IdleCloseTime checks now live in schedulertest as StateInvariant. The engine asserts it after every committed transition for both the unit tests and the Driver, and testEnv uses it through AllowStuck. The idle check now runs armed idle tasks through the handler's Validate, which subsumes the Driver's idle-route rule, so that rule and validIdleTask go. The Driver keeps its no-stuck rule: it ignores the framework visibility task and counts only valid side-effect tasks, which the shared count cannot. Co-Authored-By: Claude Fable 5.2 --- chasm/lib/scheduler/helper_test.go | 24 +- chasm/lib/scheduler/invariant_test.go | 271 ---------------- .../scheduler_stuck_regression_test.go | 5 +- chasm/lib/scheduler/schedulertest/driver.go | 29 +- chasm/lib/scheduler/schedulertest/engine.go | 83 +++-- .../lib/scheduler/schedulertest/invariants.go | 11 +- .../schedulertest/state_invariant.go | 288 ++++++++++++++++++ .../schedulertest/state_invariant_test.go | 82 +++++ .../scheduler/schedulertest/task_validity.go | 36 --- .../schedulertest/task_validity_test.go | 18 +- 10 files changed, 478 insertions(+), 369 deletions(-) delete mode 100644 chasm/lib/scheduler/invariant_test.go create mode 100644 chasm/lib/scheduler/schedulertest/state_invariant.go create mode 100644 chasm/lib/scheduler/schedulertest/state_invariant_test.go delete mode 100644 chasm/lib/scheduler/schedulertest/task_validity.go diff --git a/chasm/lib/scheduler/helper_test.go b/chasm/lib/scheduler/helper_test.go index 303aef2c960..f64ea8aeacb 100644 --- a/chasm/lib/scheduler/helper_test.go +++ b/chasm/lib/scheduler/helper_test.go @@ -81,9 +81,9 @@ type testEnv struct { MockEngine *chasm.MockEngine Logger log.Logger - // allowStuckReason, when non-empty, suppresses the stuckness invariant - // asserted by CloseTransaction. See AllowStuck. - allowStuckReason string + // state is the shared stuck and idle-deadline invariant asserted by + // CloseTransaction. See AllowStuck. + state *schedulertest.StateInvariant } // testEnvConfig holds configuration options for testEnv. @@ -194,17 +194,14 @@ func newTestEngineContextFromConfig( } // newBaseEngine builds the shared scheduler engine with this package's config, -// spec builder and stuckness invariants. +// and spec builder. The engine asserts the shared stuck and idle-deadline +// invariants after every transition. func newBaseEngine(t *testing.T, logger log.Logger, config *engineTestConfig) *schedulertest.SchedulerEngine { opts := []schedulertest.Option{ schedulertest.WithLogger(logger), schedulertest.WithSchedulerConfig(defaultConfig()), schedulertest.WithSpecBuilder(newLegacySpecBuilder(0, 0)), schedulertest.WithEngineOptions(config.engineOpts...), - schedulertest.WithSchedulerInvariant(func(t chasmtest.TB, registry *chasm.Registry, node *chasm.Node, sched *scheduler.Scheduler) { - requireNotStuck(t, node, sched) - requireIdleCloseTimeBacked(t, registry, node, sched) - }), } if config.specProcessor != nil { opts = append(opts, schedulertest.WithSpecProcessor(config.specProcessor)) @@ -362,6 +359,7 @@ func newTestEnv(t *testing.T, opts ...testEnvOption) *testEnv { Scheduler: sched, SpecProcessor: specProcessor, Logger: logger, + state: schedulertest.NewStateInvariant(registry, defaultConfig(), logger), } if config.withMockEngine { @@ -399,10 +397,7 @@ func (e *testEnv) CloseTransaction() error { if err != nil { return err } - if e.allowStuckReason == "" { - requireNotStuck(e.t, e.Node, e.Scheduler) - requireIdleCloseTimeBacked(e.t, e.Registry, e.Node, e.Scheduler) - } + e.state.Check(e.t, e.Node, e.Scheduler) return nil } @@ -415,10 +410,7 @@ func (e *testEnv) CloseTransaction() error { // finding worth reviewing. func (e *testEnv) AllowStuck(reason string) { e.t.Helper() - if reason == "" { - e.t.Fatal("AllowStuck requires a reason explaining why this test tolerates a stuck scheduler") - } - e.allowStuckReason = reason + e.state.AllowStuck(e.t, reason) } // HasTask returns true if the given task type was added with the given visibilityTime. diff --git a/chasm/lib/scheduler/invariant_test.go b/chasm/lib/scheduler/invariant_test.go deleted file mode 100644 index 699eb70cc95..00000000000 --- a/chasm/lib/scheduler/invariant_test.go +++ /dev/null @@ -1,271 +0,0 @@ -package scheduler_test - -import ( - "fmt" - "slices" - "strings" - "testing" - "time" - - "github.com/stretchr/testify/require" - "go.temporal.io/server/chasm" - "go.temporal.io/server/chasm/chasmtest" - "go.temporal.io/server/chasm/lib/scheduler" - "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" - "google.golang.org/protobuf/types/known/timestamppb" -) - -// This file implements the "stuckness" invariant for Schedules/V2. -// -// Production has repeatedly produced schedules that are open, unpaused, and -// carry no pending work: they never re-arm and never close. Every confirmed -// instance violated exactly one predicate: -// -// !sched.Closed => the tree carries at least one live logical task. -// -// Note that this is a liveness property, not a transition-validity property. -// The scheduler has no state enum to validate transitions against: -// Scheduler.LifecycleState is derived and two-valued (Completed iff Closed), -// and every sub-component hardcodes LifecycleStateRunning. In each known bug -// every transition taken was legal; the failure was that no transition -// happened at all. -// -// IMPORTANT: nothing here may touch live components. Materializing a chasm -// Field (e.g. sched.Invoker.Get(ctx)) escalates the root node's valueState and -// marks the tree dirty even for a read, which is itself one of the fault -// mechanisms behind the production bugs. So this check reads only: -// - plain Go field access on the embedded *schedulerpb.SchedulerState proto -// - the serialized snapshot returned by Node.Snapshot -// It never constructs a chasm.Context. - -// schedState is a test-only vocabulary for the scheduler's lifecycle. It is -// derived, never persisted: there is deliberately no state enum on the -// component, and adding one is a proto change with V1->V2 migration and -// rollback implications that this check does not require. -type schedState string - -const ( - // stateSentinel is a schedule-ID reservation. Inert by design. - stateSentinel schedState = "sentinel" - // stateMigrating is mid-migration to a workflow-backed (V1) scheduler, - // driven by SchedulerMigrateToWorkflowTask. - stateMigrating schedState = "migrating" - // stateCompleted is terminal: Closed is set. - stateCompleted schedState = "completed" - // stateIdle is non-terminal and quiescent, awaiting its idle timer. - stateIdle schedState = "idle" - // stateRunning is non-terminal with pending work armed. - stateRunning schedState = "running" - // stateStuck is the bug: non-terminal, but nothing will ever wake it. - stateStuck schedState = "stuck" -) - -// liveTaskCount counts logical tasks across every node in the tree. -// -// This deliberately counts *logical* tasks, from ChasmComponentAttributes. -// PureTasks and SideEffectTasks, rather than physical tasks. A physical task -// can exist on the backend while the logical task it points at has been reaped -// or invalidated, so counting physical tasks reports a schedule as armed when -// it is in fact stuck. Side-effect tasks count because some valid states, such -// as migration callback attachment, rely on one before arming pure tasks. -// (chasmtest.Engine.Tasks and testEnv.NodeBackend.TasksByCategory both expose -// physical tasks, and are the wrong source for this check.) -// -// node must be the root and the tree must be clean; Node.Snapshot panics -// otherwise. Both hold immediately after a successful CloseTransaction. -func liveTaskCount(node *chasm.Node) (int, map[string]int) { - byPath := make(map[string]int) - total := 0 - for path, n := range node.Snapshot(nil).Nodes { - attrs := n.GetMetadata().GetComponentAttributes() - taskCount := len(attrs.GetPureTasks()) + len(attrs.GetSideEffectTasks()) - if taskCount == 0 { - continue - } - byPath[path] = taskCount - total += taskCount - } - return total, byPath -} - -// derivedState computes the scheduler's lifecycle state from the tuple that -// actually represents it: (Sentinel, WorkflowMigration, Closed, IdleCloseTime, -// live logical task count). -func derivedState(sched *scheduler.Scheduler, liveTasks int) schedState { - switch { - case sched.GetSentinel(): - return stateSentinel - case sched.GetWorkflowMigration() != nil: - return stateMigrating - case sched.GetClosed(): - return stateCompleted - case liveTasks == 0: - return stateStuck - case sched.GetIdleCloseTime() != nil: - return stateIdle - default: - return stateRunning - } -} - -// idleTaskCount counts armed SchedulerIdleTasks across the tree, resolving the -// task's registered type ID rather than matching on node path (the root -// Scheduler component also carries callback and migration tasks). -func idleTaskCount(registry *chasm.Registry, node *chasm.Node) int { - wantTypeID, ok := registry.TaskIDFor(&schedulerpb.SchedulerIdleTask{}) - if !ok { - return -1 - } - count := 0 - for _, n := range node.Snapshot(nil).Nodes { - for _, task := range n.GetMetadata().GetComponentAttributes().GetPureTasks() { - if task.GetTypeId() == wantTypeID { - count++ - } - } - } - return count -} - -// requireIdleCloseTimeBacked asserts that a declared idle deadline is actually -// backed by an armed idle task. -// -// IdleCloseTime is surfaced as the ScheduleIdleCloseTime search attribute and -// is what the production stuck-schedule scanner reads. When -// SchedulerIdleTaskHandler.Validate drops a task as expiration_shift and -// nothing arms a replacement, IdleCloseTime is left behind: the schedule never -// closes, and the very signal built to detect that reports a close time that -// will never arrive. That combination is always wrong. -// -// Only asserted in this direction. Sentinels legitimately arm an idle task -// without setting IdleCloseTime (see NewSentinel), so an armed task with no -// declared deadline is not a defect. -func requireIdleCloseTimeBacked(t chasmtest.TB, registry *chasm.Registry, node *chasm.Node, sched *scheduler.Scheduler) { - t.Helper() - - if sched.GetSentinel() || sched.GetClosed() || sched.GetIdleCloseTime() == nil { - return - } - total, byPath := liveTaskCount(node) - require.NotZero(t, idleTaskCount(registry, node), - "IdleCloseTime is set to %v but no SchedulerIdleTask is armed, so the schedule will "+ - "never close and ScheduleIdleCloseTime reports a deadline that will never arrive: %s", - sched.GetIdleCloseTime().AsTime(), describeSched(sched, total, byPath)) -} - -// requireNotStuck asserts the stuckness invariant against a clean tree. -func requireNotStuck(t chasmtest.TB, node *chasm.Node, sched *scheduler.Scheduler) { - t.Helper() - - total, byPath := liveTaskCount(node) - state := derivedState(sched, total) - require.NotEqual(t, stateStuck, state, "scheduler is stuck: %s", describeSched(sched, total, byPath)) -} - -func requireValidSchedulerState( - t chasmtest.TB, - registry *chasm.Registry, - node *chasm.Node, - root chasm.RootComponent, -) { - t.Helper() - - sched, ok := root.(*scheduler.Scheduler) - require.Truef(t, ok, "scheduler test engine root has unexpected type %T", root) - requireNotStuck(t, node, sched) - requireIdleCloseTimeBacked(t, registry, node, sched) -} - -// describeSched renders the full state tuple for a failure message, so a -// triage does not require re-running under a debugger. -func describeSched(sched *scheduler.Scheduler, total int, byPath map[string]int) string { - paths := make([]string, 0, len(byPath)) - for p := range byPath { - paths = append(paths, fmt.Sprintf("%s=%d", p, byPath[p])) - } - slices.Sort(paths) - - var b strings.Builder - fmt.Fprintf(&b, "\n scheduleID: %s", sched.GetScheduleId()) - fmt.Fprintf(&b, "\n closed: %v", sched.GetClosed()) - fmt.Fprintf(&b, "\n sentinel: %v", sched.GetSentinel()) - fmt.Fprintf(&b, "\n paused: %v", sched.GetSchedule().GetState().GetPaused()) - fmt.Fprintf(&b, "\n limitedActions: %v", sched.GetSchedule().GetState().GetLimitedActions()) - fmt.Fprintf(&b, "\n remaining: %d", sched.GetSchedule().GetState().GetRemainingActions()) - fmt.Fprintf(&b, "\n conflictToken: %d", sched.GetConflictToken()) - fmt.Fprintf(&b, "\n idleCloseTime: %v", sched.GetIdleCloseTime().AsTime()) - fmt.Fprintf(&b, "\n migration: %v", sched.GetWorkflowMigration() != nil) - fmt.Fprintf(&b, "\n liveTasks: %d [%s]", total, strings.Join(paths, " ")) - return b.String() -} - -// TestDerivedState pins the classification tuple. Without this, a refactor -// that made derivedState never return stateStuck would silently disarm every -// stuckness assertion in the package while leaving the suite green. -func TestDerivedState(t *testing.T) { - t.Parallel() - - cases := []struct { - name string - state *schedulerpb.SchedulerState - liveTasks int - want schedState - }{ - { - name: "open with an armed task is running", - state: &schedulerpb.SchedulerState{}, - liveTasks: 1, - want: stateRunning, - }, - { - name: "open with an armed task and an idle deadline is idle", - state: &schedulerpb.SchedulerState{IdleCloseTime: timestamppb.New(time.Unix(100, 0))}, - liveTasks: 1, - want: stateIdle, - }, - { - name: "open with no armed task is stuck", - state: &schedulerpb.SchedulerState{}, - liveTasks: 0, - want: stateStuck, - }, - { - name: "an idle deadline does not excuse a missing task", - state: &schedulerpb.SchedulerState{IdleCloseTime: timestamppb.New(time.Unix(100, 0))}, - liveTasks: 0, - want: stateStuck, - }, - { - name: "closed with no armed task is terminal, not stuck", - state: &schedulerpb.SchedulerState{Closed: true}, - liveTasks: 0, - want: stateCompleted, - }, - { - name: "sentinel is inert by design", - state: &schedulerpb.SchedulerState{Sentinel: true}, - liveTasks: 0, - want: stateSentinel, - }, - { - name: "mid-migration is excused", - state: &schedulerpb.SchedulerState{WorkflowMigration: &schedulerpb.WorkflowMigrationState{}}, - liveTasks: 0, - want: stateMigrating, - }, - { - name: "closed takes precedence over a stale idle deadline", - state: &schedulerpb.SchedulerState{Closed: true, IdleCloseTime: timestamppb.New(time.Unix(100, 0))}, - liveTasks: 0, - want: stateCompleted, - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - sched := &scheduler.Scheduler{SchedulerState: tc.state} - require.Equal(t, tc.want, derivedState(sched, tc.liveTasks)) - }) - } -} diff --git a/chasm/lib/scheduler/scheduler_stuck_regression_test.go b/chasm/lib/scheduler/scheduler_stuck_regression_test.go index c56dfa83ba8..36cf57493dd 100644 --- a/chasm/lib/scheduler/scheduler_stuck_regression_test.go +++ b/chasm/lib/scheduler/scheduler_stuck_regression_test.go @@ -9,6 +9,7 @@ import ( "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/lib/scheduler" "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" + "go.temporal.io/server/chasm/lib/scheduler/schedulertest" "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -46,7 +47,7 @@ func TestIdleDeadlineRearmedAfterRecordingStart(t *testing.T) { armedDeadline := sched.GetIdleCloseTime() require.NotNil(t, armedDeadline, "precondition: exhausted schedule should arm an idle deadline") - require.NotZero(t, idleTaskCount(env.Registry, env.Node), "precondition: an idle task should be armed") + require.NotZero(t, schedulertest.IdleTaskCount(env.Registry, env.Node), "precondition: an idle task should be armed") // Record the start through the real invoker path. ctx2 := env.MutableContext() @@ -80,7 +81,7 @@ func TestIdleDeadlineRearmedAfterRecordingStart(t *testing.T) { // The schedule must still have a path to closing: a replacement idle task, // armed from the new deadline. - require.NotZero(t, idleTaskCount(env.Registry, env.Node), + require.NotZero(t, schedulertest.IdleTaskCount(env.Registry, env.Node), "recording a start invalidated the idle task with no replacement armed, while "+ "IdleCloseTime still advertises %v: the schedule will never close and the "+ "stuck-schedule scanner is told otherwise", sched.GetIdleCloseTime().AsTime()) diff --git a/chasm/lib/scheduler/schedulertest/driver.go b/chasm/lib/scheduler/schedulertest/driver.go index 5d902753b5c..f00984ee9b2 100644 --- a/chasm/lib/scheduler/schedulertest/driver.go +++ b/chasm/lib/scheduler/schedulertest/driver.go @@ -52,6 +52,7 @@ type startFaults struct { // re-arms a generator task), so callers bound the run with maxSteps; quiescence is // reached only for paused/exhausted/closed schedules. type Driver struct { + base *SchedulerEngine recordCalls chan *RecordCall external *externalLedger startCalls chan *StartCall @@ -118,6 +119,7 @@ type driverConfig struct { maxLag time.Duration maxRun time.Duration controlledStarts bool + noStateCheck bool } // DriverOption configures a Driver at construction. @@ -145,6 +147,14 @@ func WithConfig(cfg *scheduler.Config) DriverOption { return func(c *driverConfig) { c.config = cfg } } +// WithoutStateInvariant leaves the shared stuck and idle-deadline invariants +// off the engine, which otherwise fails the test the moment a transition commits +// a violating state. Use it only in a test that corrupts the scheduler on +// purpose, and assert Driver.StateViolations instead. +func WithoutStateInvariant() DriverOption { + return func(c *driverConfig) { c.noStateCheck = true } +} + // WithMaxSideEffectLag bounds how long StepPure may leave a side-effect task // (e.g. InvokerExecuteTask) undispatched while moving the clock forward. func WithMaxSideEffectLag(lag time.Duration) DriverOption { @@ -297,18 +307,23 @@ func newDriverScaffold(t chasmtest.TB, opts ...DriverOption) (*Driver, *driverCo metricsHandler := metricstest.NewCaptureHandler() capture := metricsHandler.StartCapture() - base := NewEngine(t, + engineOpts := []Option{ WithLogger(logger), WithSchedulerConfig(cfg.config), WithTimeSource(ts), WithInvokerTimeSource(ts), WithInvokerMetrics(metricsHandler), WithClients(frontend, history), - ) + } + if cfg.noStateCheck { + engineOpts = append(engineOpts, WithoutStateCheck()) + } + base := NewEngine(t, engineOpts...) engine := base.Engine registry := base.Registry d := &Driver{ + base: base, recordCalls: make(chan *RecordCall), external: external, startCalls: startCalls, recorded: make(map[string]ExternalStart), replacingSeen: make(map[string]struct{}), @@ -350,6 +365,14 @@ func (d *Driver) startExecution(startFn func(chasm.MutableContext) (*scheduler.S d.ReadScheduler(func(s *scheduler.Scheduler, _ chasm.Context) { d.initialActionCount = s.Info.GetActionCount() }) } +// StateViolations evaluates the shared stuck and idle-deadline invariants +// against the last committed state without failing the test. See +// [WithoutStateInvariant]. +func (d *Driver) StateViolations() []string { + d.T.Helper() + return d.base.StateViolations() +} + // Now returns the engine's current clock time. func (d *Driver) Now() time.Time { return d.TimeSource.Now() @@ -652,7 +675,6 @@ func (d *Driver) MigrateToWorkflow() error { // step's CloseTransaction). It is the input to invariant checks. type Snapshot struct { UntransferableActions int - HasValidIdleTask bool AcceptedStarts map[string]ExternalStart RecordedStarts map[string]ExternalStart ActionCount int64 @@ -736,7 +758,6 @@ func (d *Driver) Snapshot() Snapshot { DroppedStarts: d.droppedStarts(), } d.ReadScheduler(func(s *scheduler.Scheduler, ctx chasm.Context) { - snap.HasValidIdleTask = d.validIdleTask(s, ctx) snap.ActionCount = s.Info.GetActionCount() snap.Closed = s.Closed snap.IsSentinel = s.IsSentinel() diff --git a/chasm/lib/scheduler/schedulertest/engine.go b/chasm/lib/scheduler/schedulertest/engine.go index ac0f9373869..cdf968b3036 100644 --- a/chasm/lib/scheduler/schedulertest/engine.go +++ b/chasm/lib/scheduler/schedulertest/engine.go @@ -32,6 +32,9 @@ type SchedulerEngine struct { Engine *chasmtest.Engine Registry *chasm.Registry Logger log.Logger + // State is the stuck and idle-deadline invariant the engine asserts after + // every committed transition (unless built WithoutStateCheck). + State *StateInvariant // Config is the scheduler config the library was wired with. Config *scheduler.Config // TimeSource is the clock the engine was built with, or nil if the engine @@ -44,14 +47,19 @@ type SchedulerEngine struct { Ctx context.Context Key chasm.ExecutionKey Ref chasm.ComponentRef + + // node and root are the tree and Scheduler as of the last committed + // transition, for StateViolations. + node *chasm.Node + root *scheduler.Scheduler } type engineOptions struct { - logger log.Logger - library LibraryOptions - engineOpts []chasmtest.EngineOption - timeSource *clock.EventTimeSource - invariants []func(chasmtest.TB, *chasm.Registry, *chasm.Node, *scheduler.Scheduler) + logger log.Logger + library LibraryOptions + engineOpts []chasmtest.EngineOption + timeSource *clock.EventTimeSource + noStateCheck bool } // Option configures [NewEngine]. @@ -114,10 +122,11 @@ func WithEngineOptions(opts ...chasmtest.EngineOption) Option { return func(o *engineOptions) { o.engineOpts = append(o.engineOpts, opts...) } } -// WithSchedulerInvariant installs a check run after every successful transition -// with the Scheduler root. It is skipped for roots that are not a Scheduler. -func WithSchedulerInvariant(check func(chasmtest.TB, *chasm.Registry, *chasm.Node, *scheduler.Scheduler)) Option { - return func(o *engineOptions) { o.invariants = append(o.invariants, check) } +// WithoutStateCheck leaves the stuck and idle-deadline invariants off the +// engine. Only a test that corrupts the scheduler on purpose and asserts +// [StateInvariant.Violations] catches it should need this. +func WithoutStateCheck() Option { + return func(o *engineOptions) { o.noStateCheck = true } } // NewEngine builds a registry with the core and scheduler libraries, wraps it in @@ -151,34 +160,48 @@ func NewEngine(t chasmtest.TB, opts ...Option) *SchedulerEngine { t.Fatalf("failed to register scheduler library: %v", err) } - engineOpts := o.engineOpts - for _, check := range o.invariants { - engineOpts = append(engineOpts, chasmtest.WithInvariantCheck( - func(t chasmtest.TB, node *chasm.Node, root chasm.RootComponent) { - t.Helper() - sched, ok := root.(*scheduler.Scheduler) - if !ok { - t.Fatalf("scheduler test engine root has unexpected type %T", root) - } - check(t, registry, node, sched) - }, - )) - } - engine := chasmtest.NewEngine(t, registry, engineOpts...) - - key := chasm.ExecutionKey{NamespaceID: NamespaceID, BusinessID: ScheduleID} - return &SchedulerEngine{ + state := NewStateInvariant(registry, o.library.Config, o.logger) + se := &SchedulerEngine{ T: t, - Engine: engine, Registry: registry, + State: state, Logger: o.logger, Config: o.library.Config, TimeSource: o.timeSource, SpecProcessor: o.library.SpecProcessor, - Ctx: chasm.NewEngineContext(context.Background(), engine), - Key: key, - Ref: chasm.NewComponentRef[*scheduler.Scheduler](key), } + engineOpts := append(o.engineOpts, chasmtest.WithInvariantCheck( + func(t chasmtest.TB, node *chasm.Node, root chasm.RootComponent) { + t.Helper() + sched, ok := root.(*scheduler.Scheduler) + if !ok { + t.Fatalf("scheduler test engine root has unexpected type %T", root) + } + se.node, se.root = node, sched + if !o.noStateCheck { + state.Check(t, node, sched) + } + }, + )) + engine := chasmtest.NewEngine(t, registry, engineOpts...) + + se.Engine = engine + se.Key = chasm.ExecutionKey{NamespaceID: NamespaceID, BusinessID: ScheduleID} + se.Ref = chasm.NewComponentRef[*scheduler.Scheduler](se.Key) + se.Ctx = chasm.NewEngineContext(context.Background(), engine) + return se +} + +// StateViolations evaluates the stuck and idle-deadline invariants against the +// last committed state without failing the test. It is how a test built +// [WithoutStateCheck] asserts that the check catches state it corrupted on +// purpose. +func (e *SchedulerEngine) StateViolations() []string { + e.T.Helper() + if e.node == nil { + e.T.Fatalf("no transition has committed yet") + } + return e.State.Violations(e.node, e.root) } // Start starts the scheduler execution with startFn, recording [CreateRequestID]. diff --git a/chasm/lib/scheduler/schedulertest/invariants.go b/chasm/lib/scheduler/schedulertest/invariants.go index 394b31101ee..16c089415a7 100644 --- a/chasm/lib/scheduler/schedulertest/invariants.go +++ b/chasm/lib/scheduler/schedulertest/invariants.go @@ -40,6 +40,14 @@ func CheckInvariants(prev *Snapshot, cur Snapshot) []Violation { // progress. The only taskless, non-closed state allowed is "held open" // (paused or pending backfill). When idle is armed there is a pending idle // task, so HasPendingTask covers that case. + // + // The engine also asserts a stuck check on every commit (StateInvariant), + // but this rule is stronger in one way and is kept alongside it: + // HasPendingTask counts only side-effect tasks whose handler still considers + // them valid and ignores the framework visibility task, whereas the engine + // check counts every logical task, including the visibility task that a test + // engine never dispatches. The engine check is stronger in the other + // direction (no held-open exemption), which is why both run. if !cur.Closed && !cur.HasPendingTask && !cur.IsHeldOpen { vs.add("no-stuck", "scheduler has no pending task, is not closed, and is not held open "+ @@ -78,9 +86,6 @@ func CheckInvariants(prev *Snapshot, cur Snapshot) []Violation { vs = append(vs, checkLedger(prev, cur)...) vs = append(vs, checkStartAccounting(cur)...) - if !cur.Closed && !cur.IdleCloseTime.IsZero() && !cur.HasValidIdleTask { - vs.add("idle-route", "advertised idle deadline %s has no valid logical idle task", cur.IdleCloseTime) - } if prev != nil { // (3) Closed is terminal: a closed schedule must never reopen. diff --git a/chasm/lib/scheduler/schedulertest/state_invariant.go b/chasm/lib/scheduler/schedulertest/state_invariant.go new file mode 100644 index 00000000000..ef16bb9e3ee --- /dev/null +++ b/chasm/lib/scheduler/schedulertest/state_invariant.go @@ -0,0 +1,288 @@ +package schedulertest + +import ( + "context" + "fmt" + "slices" + "strings" + "sync" + + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/chasmtest" + "go.temporal.io/server/chasm/lib/scheduler" + schedulerpb "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/metrics" + "google.golang.org/protobuf/proto" +) + +// This file implements the "stuckness" and idle-deadline invariants for +// Schedules/V2. [SchedulerEngine] installs them on every transition it commits, +// and the unit tests' testEnv asserts them on every CloseTransaction, so the +// whole suite and the lifecycle property check the same two predicates. +// +// Production has repeatedly produced schedules that are open, unpaused, and +// carry no pending work: they never re-arm and never close. Every confirmed +// instance violated exactly one predicate: +// +// !sched.Closed => the tree carries at least one live logical task. +// +// This is a liveness property, not a transition-validity property. The +// scheduler has no state enum to validate transitions against: +// Scheduler.LifecycleState is derived and two-valued (Completed iff Closed), +// and every sub-component hardcodes LifecycleStateRunning. In each known bug +// every transition taken was legal; the failure was that no transition +// happened at all. +// +// The stuck predicate reads only plain Go field access on the embedded +// *schedulerpb.SchedulerState proto and the serialized snapshot returned by +// Node.Snapshot. It never constructs a chasm.Context, because materializing a +// chasm Field (e.g. sched.Invoker.Get(ctx)) escalates the root node's +// valueState and marks the tree dirty even for a read. Only the idle-task +// validity check does, see idleTaskValid. + +// schedState is a test-only vocabulary for the scheduler's lifecycle. It is +// derived, never persisted: there is deliberately no state enum on the +// component, and adding one is a proto change with V1->V2 migration and +// rollback implications that this check does not require. +type schedState string + +const ( + // stateSentinel is a schedule-ID reservation. Inert by design. + stateSentinel schedState = "sentinel" + // stateMigrating is mid-migration to a workflow-backed (V1) scheduler, + // driven by SchedulerMigrateToWorkflowTask. + stateMigrating schedState = "migrating" + // stateCompleted is terminal: Closed is set. + stateCompleted schedState = "completed" + // stateIdle is non-terminal and quiescent, awaiting its idle timer. + stateIdle schedState = "idle" + // stateRunning is non-terminal with pending work armed. + stateRunning schedState = "running" + // stateStuck is the bug: non-terminal, but nothing will ever wake it. + stateStuck schedState = "stuck" +) + +// StateInvariant asserts the stuck and idle-deadline invariants against a clean +// tree. Build one with [NewStateInvariant]; [NewEngine] installs one on the +// engine it builds and exposes it as [SchedulerEngine.State]. +type StateInvariant struct { + registry *chasm.Registry + idleHandler *scheduler.SchedulerIdleTaskHandler + + mu sync.Mutex + allowStuckReason string +} + +// NewStateInvariant builds the check for trees registered in registry. +// config and logger back the idle task handler used to validate armed idle +// tasks. +func NewStateInvariant(registry *chasm.Registry, config *scheduler.Config, logger log.Logger) *StateInvariant { + return &StateInvariant{ + registry: registry, + idleHandler: scheduler.NewSchedulerIdleTaskHandler(scheduler.SchedulerIdleTaskHandlerOptions{ + Config: config, MetricsHandler: metrics.NoopMetricsHandler, BaseLogger: logger, + }), + } +} + +// AllowStuck opts out of both checks, for tests that deliberately drive the +// scheduler into a state with no pending work. +// +// reason is mandatory: an unexplained opt-out is indistinguishable from a +// silently tolerated bug, and the set of tests that need one is itself a +// finding worth reviewing. +func (c *StateInvariant) AllowStuck(t chasmtest.TB, reason string) { + t.Helper() + if reason == "" { + t.Fatal("AllowStuck requires a reason explaining why this test tolerates a stuck scheduler") + } + c.mu.Lock() + defer c.mu.Unlock() + c.allowStuckReason = reason +} + +// Check fails t if the scheduler in a clean tree is stuck, or advertises an idle +// deadline that no valid idle task backs. node must be the root and the tree +// must be clean; both hold immediately after a successful CloseTransaction. +func (c *StateInvariant) Check(t chasmtest.TB, node *chasm.Node, sched *scheduler.Scheduler) { + t.Helper() + if vs := c.Violations(node, sched); len(vs) > 0 { + t.Fatalf("%s", strings.Join(vs, "\n")) + } +} + +// Violations reports every broken predicate without failing a test, so a test +// that corrupts state on purpose can assert the check catches it. It reports +// nothing once [StateInvariant.AllowStuck] was called. +func (c *StateInvariant) Violations(node *chasm.Node, sched *scheduler.Scheduler) []string { + c.mu.Lock() + allowed := c.allowStuckReason != "" + c.mu.Unlock() + if allowed { + return nil + } + + var out []string + total, byPath := liveTaskCount(node) + if derivedState(sched, total) == stateStuck { + out = append(out, fmt.Sprintf("scheduler is stuck: %s", describeSched(sched, total, byPath))) + } + if v := c.idleViolation(node, sched, total, byPath); v != "" { + out = append(out, v) + } + return out +} + +// idleViolation asserts that a declared idle deadline is backed by an armed +// idle task that its handler still considers valid. +// +// IdleCloseTime is surfaced as the ScheduleIdleCloseTime search attribute and +// is what the production stuck-schedule scanner reads. When +// SchedulerIdleTaskHandler.Validate drops a task as expiration_shift and +// nothing arms a replacement, IdleCloseTime is left behind: the schedule never +// closes, and the very signal built to detect that reports a close time that +// will never arrive. Counting armed tasks is not enough, since an idle task the +// handler drops (deadline moved, schedule held open) leaves the same hole, so +// the armed tasks are run through Validate. +// +// Only asserted in this direction. Sentinels legitimately arm an idle task +// without setting IdleCloseTime (see NewSentinel), so an armed task with no +// declared deadline is not a defect. +func (c *StateInvariant) idleViolation(node *chasm.Node, sched *scheduler.Scheduler, total int, byPath map[string]int) string { + if sched.GetSentinel() || sched.GetClosed() || sched.GetIdleCloseTime() == nil { + return "" + } + if c.idleTaskValid(node, sched) { + return "" + } + return fmt.Sprintf("IdleCloseTime is set to %v but no valid SchedulerIdleTask is armed (%d armed), so the "+ + "schedule will never close and ScheduleIdleCloseTime reports a deadline that will never arrive: %s", + sched.GetIdleCloseTime().AsTime(), IdleTaskCount(c.registry, node), describeSched(sched, total, byPath)) +} + +// idleTaskValid reports whether any armed idle task passes the handler's +// Validate. It validates through a read-only context over the clean tree. +func (c *StateInvariant) idleTaskValid(node *chasm.Node, sched *scheduler.Scheduler) bool { + wantTypeID, ok := c.registry.TaskIDFor(&schedulerpb.SchedulerIdleTask{}) + if !ok { + return false + } + ctx := chasm.NewContext(context.Background(), node) + for _, n := range node.Snapshot(nil).Nodes { + for _, task := range n.GetMetadata().GetComponentAttributes().GetPureTasks() { + if task.GetTypeId() != wantTypeID { + continue + } + value := &schedulerpb.SchedulerIdleTask{} + if err := proto.Unmarshal(task.GetData().GetData(), value); err != nil { + return false + } + valid, err := c.idleHandler.Validate(ctx, sched, chasm.TaskInvocation{ + TaskAttributes: chasm.TaskAttributes{ScheduledTime: task.GetScheduledTime().AsTime()}, + }, value) + if err == nil && valid { + return true + } + } + } + return false +} + +// liveTaskCount counts logical tasks across every node in the tree. +// +// This deliberately counts *logical* tasks, from ChasmComponentAttributes. +// PureTasks and SideEffectTasks, rather than physical tasks. A physical task +// can exist on the backend while the logical task it points at has been reaped +// or invalidated, so counting physical tasks reports a schedule as armed when +// it is in fact stuck. Side-effect tasks count because some valid states, such +// as migration callback attachment, rely on one before arming pure tasks. +// (chasmtest.Engine.Tasks and testEnv.NodeBackend.TasksByCategory both expose +// physical tasks, and are the wrong source for this check.) +// +// Note that the framework-managed visibility task counts here, as it always +// has. A test engine never dispatches it, so it stays in the tree after any +// transition that touches a search attribute and masks a scheduler with nothing +// else left to run. Excluding it exposes a hand-built taskless state in +// TestGeneratorTask_EngineHighWaterMarkInvalidation, so the lifecycle driver +// keeps its own no-stuck rule, which ignores it (see CheckInvariants). +// +// node must be the root and the tree must be clean; Node.Snapshot panics +// otherwise. Both hold immediately after a successful CloseTransaction. +func liveTaskCount(node *chasm.Node) (int, map[string]int) { + byPath := make(map[string]int) + total := 0 + for path, n := range node.Snapshot(nil).Nodes { + attrs := n.GetMetadata().GetComponentAttributes() + taskCount := len(attrs.GetPureTasks()) + len(attrs.GetSideEffectTasks()) + if taskCount == 0 { + continue + } + byPath[path] = taskCount + total += taskCount + } + return total, byPath +} + +// derivedState computes the scheduler's lifecycle state from the tuple that +// actually represents it: (Sentinel, WorkflowMigration, Closed, IdleCloseTime, +// live logical task count). +func derivedState(sched *scheduler.Scheduler, liveTasks int) schedState { + switch { + case sched.GetSentinel(): + return stateSentinel + case sched.GetWorkflowMigration() != nil: + return stateMigrating + case sched.GetClosed(): + return stateCompleted + case liveTasks == 0: + return stateStuck + case sched.GetIdleCloseTime() != nil: + return stateIdle + default: + return stateRunning + } +} + +// IdleTaskCount counts armed SchedulerIdleTasks across the tree, resolving the +// task's registered type ID rather than matching on node path (the root +// Scheduler component also carries callback and migration tasks). It returns -1 +// if the idle task is not registered. +func IdleTaskCount(registry *chasm.Registry, node *chasm.Node) int { + wantTypeID, ok := registry.TaskIDFor(&schedulerpb.SchedulerIdleTask{}) + if !ok { + return -1 + } + count := 0 + for _, n := range node.Snapshot(nil).Nodes { + for _, task := range n.GetMetadata().GetComponentAttributes().GetPureTasks() { + if task.GetTypeId() == wantTypeID { + count++ + } + } + } + return count +} + +// describeSched renders the full state tuple for a failure message, so a +// triage does not require re-running under a debugger. +func describeSched(sched *scheduler.Scheduler, total int, byPath map[string]int) string { + paths := make([]string, 0, len(byPath)) + for p := range byPath { + paths = append(paths, fmt.Sprintf("%s=%d", p, byPath[p])) + } + slices.Sort(paths) + + var b strings.Builder + fmt.Fprintf(&b, "\n scheduleID: %s", sched.GetScheduleId()) + fmt.Fprintf(&b, "\n closed: %v", sched.GetClosed()) + fmt.Fprintf(&b, "\n sentinel: %v", sched.GetSentinel()) + fmt.Fprintf(&b, "\n paused: %v", sched.GetSchedule().GetState().GetPaused()) + fmt.Fprintf(&b, "\n limitedActions: %v", sched.GetSchedule().GetState().GetLimitedActions()) + fmt.Fprintf(&b, "\n remaining: %d", sched.GetSchedule().GetState().GetRemainingActions()) + fmt.Fprintf(&b, "\n conflictToken: %d", sched.GetConflictToken()) + fmt.Fprintf(&b, "\n idleCloseTime: %v", sched.GetIdleCloseTime().AsTime()) + fmt.Fprintf(&b, "\n migration: %v", sched.GetWorkflowMigration() != nil) + fmt.Fprintf(&b, "\n liveTasks: %d [%s]", total, strings.Join(paths, " ")) + return b.String() +} diff --git a/chasm/lib/scheduler/schedulertest/state_invariant_test.go b/chasm/lib/scheduler/schedulertest/state_invariant_test.go new file mode 100644 index 00000000000..71351b521b7 --- /dev/null +++ b/chasm/lib/scheduler/schedulertest/state_invariant_test.go @@ -0,0 +1,82 @@ +package schedulertest + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.temporal.io/server/chasm/lib/scheduler" + schedulerpb "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// TestDerivedState pins the classification tuple. Without this, a refactor +// that made derivedState never return stateStuck would silently disarm every +// stuckness assertion in the package while leaving the suite green. +func TestDerivedState(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + state *schedulerpb.SchedulerState + liveTasks int + want schedState + }{ + { + name: "open with an armed task is running", + state: &schedulerpb.SchedulerState{}, + liveTasks: 1, + want: stateRunning, + }, + { + name: "open with an armed task and an idle deadline is idle", + state: &schedulerpb.SchedulerState{IdleCloseTime: timestamppb.New(time.Unix(100, 0))}, + liveTasks: 1, + want: stateIdle, + }, + { + name: "open with no armed task is stuck", + state: &schedulerpb.SchedulerState{}, + liveTasks: 0, + want: stateStuck, + }, + { + name: "an idle deadline does not excuse a missing task", + state: &schedulerpb.SchedulerState{IdleCloseTime: timestamppb.New(time.Unix(100, 0))}, + liveTasks: 0, + want: stateStuck, + }, + { + name: "closed with no armed task is terminal, not stuck", + state: &schedulerpb.SchedulerState{Closed: true}, + liveTasks: 0, + want: stateCompleted, + }, + { + name: "sentinel is inert by design", + state: &schedulerpb.SchedulerState{Sentinel: true}, + liveTasks: 0, + want: stateSentinel, + }, + { + name: "mid-migration is excused", + state: &schedulerpb.SchedulerState{WorkflowMigration: &schedulerpb.WorkflowMigrationState{}}, + liveTasks: 0, + want: stateMigrating, + }, + { + name: "closed takes precedence over a stale idle deadline", + state: &schedulerpb.SchedulerState{Closed: true, IdleCloseTime: timestamppb.New(time.Unix(100, 0))}, + liveTasks: 0, + want: stateCompleted, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + sched := &scheduler.Scheduler{SchedulerState: tc.state} + require.Equal(t, tc.want, derivedState(sched, tc.liveTasks)) + }) + } +} diff --git a/chasm/lib/scheduler/schedulertest/task_validity.go b/chasm/lib/scheduler/schedulertest/task_validity.go deleted file mode 100644 index 72f4bb21be3..00000000000 --- a/chasm/lib/scheduler/schedulertest/task_validity.go +++ /dev/null @@ -1,36 +0,0 @@ -package schedulertest - -import ( - "go.temporal.io/server/chasm" - "go.temporal.io/server/chasm/lib/scheduler" - schedulerpb "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" - "go.temporal.io/server/common/metrics" - "google.golang.org/protobuf/proto" -) - -func (d *Driver) validIdleTask(s *scheduler.Scheduler, ctx chasm.Context) bool { - id, ok := d.Registry.TaskIDFor(&schedulerpb.SchedulerIdleTask{}) - if !ok { - d.T.Fatal("idle task is not registered") - } - handler := scheduler.NewSchedulerIdleTaskHandler(scheduler.SchedulerIdleTaskHandlerOptions{ - Config: d.Config, MetricsHandler: metrics.NoopMetricsHandler, BaseLogger: d.Logger, - }) - for _, task := range d.runner.PendingPureTasks() { - if task.GetTypeId() != id { - continue - } - value := &schedulerpb.SchedulerIdleTask{} - if err := proto.Unmarshal(task.GetData().GetData(), value); err != nil { - d.T.Fatalf("decode idle task: %v", err) - } - valid, err := handler.Validate(ctx, s, chasm.TaskInvocation{TaskAttributes: chasm.TaskAttributes{ScheduledTime: task.GetScheduledTime().AsTime()}}, value) - if err != nil { - d.T.Fatalf("validate idle task: %v", err) - } - if valid { - return true - } - } - return false -} diff --git a/chasm/lib/scheduler/schedulertest/task_validity_test.go b/chasm/lib/scheduler/schedulertest/task_validity_test.go index ce37d4ed648..5f8a4724544 100644 --- a/chasm/lib/scheduler/schedulertest/task_validity_test.go +++ b/chasm/lib/scheduler/schedulertest/task_validity_test.go @@ -62,12 +62,17 @@ func TestOldExecuteTaskCanProcessCurrentWork(t *testing.T) { require.Contains(t, d.Snapshot().AcceptedStarts, "current-request") } -func TestIdleRouteRequiresMatchingLogicalTask(t *testing.T) { +// TestIdleDeadlineRequiresValidIdleTask corrupts the advertised idle deadline so +// the armed idle task no longer matches it, and asserts the shared state +// invariant (which validates armed idle tasks through the handler) reports it. +// The driver is built WithoutStateInvariant, since the engine would otherwise +// fail the test the moment the corrupting transition commits. +func TestIdleDeadlineRequiresValidIdleTask(t *testing.T) { t.Parallel() schedule := DefaultSchedule() schedule.State.LimitedActions = true - d := NewDriver(t, WithSchedule(schedule)) - require.True(t, d.Snapshot().HasValidIdleTask) + d := NewDriver(t, WithSchedule(schedule), WithoutStateInvariant()) + require.Empty(t, d.StateViolations()) _, err := d.Engine.UpdateComponent(chasm.NewEngineContext(context.Background(), d.Engine), d.ref, func(ctx chasm.MutableContext, c chasm.Component) error { s := c.(*scheduler.Scheduler) @@ -77,10 +82,9 @@ func TestIdleRouteRequiresMatchingLogicalTask(t *testing.T) { return nil }) require.NoError(t, err) - snap := d.Snapshot() - require.False(t, snap.HasValidIdleTask) - violations := CheckInvariants(nil, snap) - require.Contains(t, violations, Violation{Name: "idle-route", Message: "advertised idle deadline " + snap.IdleCloseTime.String() + " has no valid logical idle task"}) + violations := d.StateViolations() + require.Len(t, violations, 1) + require.Contains(t, violations[0], "no valid SchedulerIdleTask is armed") } func TestRetryTaskEqualityAndRearming(t *testing.T) { From 7c1bbc1a75bbc954b8ba5a05aa1068ae011103f3 Mon Sep 17 00:00:00 2001 From: chaptersix <13949480+chaptersix@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:37:43 -0600 Subject: [PATCH 21/22] Build the scheduler driver on the shared test engine Driver now embeds SchedulerEngine for the engine, registry, clock and typed read and update helpers, and keeps only what is lifecycle specific. The fake frontend and history services move to fakes.go, Snapshot to snapshot.go, and Patch, Update, CompleteWorkflow, MigrateToWorkflow and DeliverCompletion go through the shared update helper. Public method names are unchanged. Co-Authored-By: Claude Fable 5.2 --- chasm/lib/scheduler/schedulertest/driver.go | 624 ++++-------------- chasm/lib/scheduler/schedulertest/external.go | 16 - chasm/lib/scheduler/schedulertest/fakes.go | 173 +++++ .../scheduler/schedulertest/inflight_test.go | 2 +- chasm/lib/scheduler/schedulertest/snapshot.go | 172 +++++ .../schedulertest/task_validity_test.go | 8 +- 6 files changed, 489 insertions(+), 506 deletions(-) create mode 100644 chasm/lib/scheduler/schedulertest/fakes.go create mode 100644 chasm/lib/scheduler/schedulertest/snapshot.go diff --git a/chasm/lib/scheduler/schedulertest/driver.go b/chasm/lib/scheduler/schedulertest/driver.go index f00984ee9b2..e73f2738a97 100644 --- a/chasm/lib/scheduler/schedulertest/driver.go +++ b/chasm/lib/scheduler/schedulertest/driver.go @@ -1,22 +1,13 @@ package schedulertest import ( - "cmp" - "context" "errors" - "fmt" - "maps" - "sync" "time" commonpb "go.temporal.io/api/common/v1" - enumspb "go.temporal.io/api/enums/v1" schedulepb "go.temporal.io/api/schedule/v1" "go.temporal.io/api/serviceerror" - workflowpb "go.temporal.io/api/workflow/v1" "go.temporal.io/api/workflowservice/v1" - "go.temporal.io/server/api/historyservice/v1" - "go.temporal.io/server/api/historyservicemock/v1" persistencespb "go.temporal.io/server/api/persistence/v1" schedulespb "go.temporal.io/server/api/schedule/v1" "go.temporal.io/server/chasm" @@ -25,82 +16,49 @@ import ( schedulerpb "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" "go.temporal.io/server/common" "go.temporal.io/server/common/clock" - "go.temporal.io/server/common/log" - "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/metrics/metricstest" - "go.temporal.io/server/common/testing/mockapi/workflowservicemock/v1" "go.temporal.io/server/common/testing/testlogger" "go.uber.org/mock/gomock" - "google.golang.org/grpc" "google.golang.org/protobuf/types/known/timestamppb" ) -// startFaults counts the upcoming StartWorkflowExecution RPCs that fail. -type startFaults struct { - rejected int // the RPC fails without the start being accepted - lostReply int // the start is accepted but the response is lost -} - -// Driver builds a real CHASM scheduler on top of a [chasmtest.Engine] and steps -// it forward through every task it schedules for itself, advancing a controllable -// clock. After each settled transition it invokes AfterStep (if set), which is -// where invariant assertions are plugged in. +// Driver steps a real CHASM scheduler through every task it schedules for +// itself, advancing a controllable clock. It is a [SchedulerEngine] plus what is +// specific to driving the scheduler's lifecycle: fake frontend and history +// services with an external ledger and fault injection, the hooks that observe +// each step, and the operations a customer or the system performs on a schedule. +// After each settled transition it invokes AfterStep (if set), which is where +// invariant assertions are plugged in. // // A "step", run by a [chasmtest.TaskRunner], fires every pure task due at the // current reference time and every side-effect task whose visibility time has -// arrived, then advances the clock to the next scheduled task. A healthy interval schedule never truly quiesces (it always -// re-arms a generator task), so callers bound the run with maxSteps; quiescence is -// reached only for paused/exhausted/closed schedules. +// arrived, then advances the clock to the next scheduled task. A healthy interval +// schedule never truly quiesces (it always re-arms a generator task), so callers +// bound the run with maxSteps; quiescence is reached only for paused, exhausted +// or closed schedules. type Driver struct { - base *SchedulerEngine + *SchedulerEngine + *fakeServices + recordCalls chan *RecordCall - external *externalLedger - startCalls chan *StartCall - actionsBefore int recorded map[string]ExternalStart replacingSeen map[string]struct{} initialActionCount int64 - T chasmtest.TB - Engine *chasmtest.Engine - Registry *chasm.Registry - TimeSource *clock.EventTimeSource - Logger log.Logger - - Frontend *workflowservicemock.MockWorkflowServiceClient - History *historyservicemock.MockHistoryServiceClient - - // Config is the scheduler config the library was wired with. Snapshot reads - // Tweakables (MaxBufferSize, MaxActionsPerExecution) from it so the - // buffer-bound and action-budget invariants can be evaluated. - Config *scheduler.Config - - ref chasm.ComponentRef - execKey chasm.ExecutionKey + // metricsCapture records the scheduler's metrics; with the start ledger it + // backs the accounting invariants. + metricsCapture *metricstest.Capture // runner steps the execution through its pure and side-effect tasks. runner *chasmtest.TaskRunner - // actionRPCs points at the running count of action RPCs (StartWorkflowExecution - // + TerminateWorkflowExecution + RequestCancelWorkflowExecution) made through - // the mock clients. It is allocated before the clients so the stub closures can - // increment it. maxActionsObserved is the high-water mark of action RPCs made by - // any single side-effect task dispatch (one InvokerExecuteTask), which backs the + // actionsBefore is the action RPC count when the current side-effect task + // began. maxActionsObserved is the high-water mark of action RPCs made by any + // single side-effect task dispatch (one InvokerExecuteTask), which backs the // action-budget invariant. - actionRPCs *int + actionsBefore int maxActionsObserved int - // startRequests records the RequestId of every StartWorkflowExecution RPC, and - // metricsCapture the scheduler's metrics; together they back the start ledger. - startRequests map[string]struct{} - metricsCapture *metricstest.Capture - // rpcMu guards actionRPCs and startRequests: InvokerExecuteTask issues - // starts from concurrent goroutines. - rpcMu *sync.Mutex - - // faults injects failures into uncontrolled start RPCs. Guarded by rpcMu. - faults *startFaults - // maxRunDuration auto-completes a started workflow before the clock moves // past its StartTime plus this. Zero means workflows only complete through // CompleteWorkflow. @@ -173,17 +131,16 @@ func WithMaxRunDuration(d time.Duration) DriverOption { func NewDriver(t chasmtest.TB, opts ...DriverOption) *Driver { t.Helper() d, cfg := newDriverScaffold(t, opts...) - req := &schedulerpb.CreateScheduleRequest{ NamespaceId: NamespaceID, FrontendRequest: &workflowservice.CreateScheduleRequest{ Namespace: Namespace, ScheduleId: ScheduleID, Schedule: cfg.schedule, - RequestId: "create-req", + RequestId: CreateRequestID, }, } - d.startExecution(func(mctx chasm.MutableContext) (*scheduler.Scheduler, error) { + d.start(func(mctx chasm.MutableContext) (*scheduler.Scheduler, error) { return scheduler.CreateScheduler(mctx, req) }) return d @@ -195,14 +152,14 @@ func NewDriver(t chasmtest.TB, opts ...DriverOption) *Driver { func NewDriverFromMigration(t chasmtest.TB, req *schedulerpb.CreateFromMigrationStateRequest, opts ...DriverOption) *Driver { t.Helper() d, _ := newDriverScaffold(t, opts...) - d.startExecution(func(mctx chasm.MutableContext) (*scheduler.Scheduler, error) { + d.start(func(mctx chasm.MutableContext) (*scheduler.Scheduler, error) { return scheduler.CreateSchedulerFromMigration(mctx, req) }) return d } -// newDriverScaffold builds the engine, registry, clients, and time source, but -// does not start an execution. +// newDriverScaffold builds the engine, fake services and clock, but does not +// start an execution. func newDriverScaffold(t chasmtest.TB, opts ...DriverOption) (*Driver, *driverConfig) { t.Helper() @@ -215,164 +172,46 @@ func newDriverScaffold(t chasmtest.TB, opts ...DriverOption) (*Driver, *driverCo opt(cfg) } - ctrl := gomock.NewController(t) - logger := testlogger.NewTestLogger(t, testlogger.FailOnExpectedErrorOnly) - ts := clock.NewEventTimeSource() ts.Update(cfg.startTime) - - frontend := workflowservicemock.NewMockWorkflowServiceClient(ctrl) - history := historyservicemock.NewMockHistoryServiceClient(ctrl) - - // actionRPCs counts every action RPC (start/terminate/cancel) the invoker - // makes through the mock clients, so Step can measure how many actions a single - // ExecuteTask takes (the action-budget invariant). - actionRPCs := new(int) - startRequests := make(map[string]struct{}) - external := &externalLedger{starts: make(map[string]ExternalStart)} - var startCalls chan *StartCall - if cfg.controlledStarts { - startCalls = make(chan *StartCall) - } - rpcMu := &sync.Mutex{} - faults := &startFaults{} - countAction := func() { - rpcMu.Lock() - defer rpcMu.Unlock() - *actionRPCs++ - } - - // Default happy-path stubs; tests can layer more specific expectations. - frontend.EXPECT().StartWorkflowExecution(gomock.Any(), gomock.Any()). - DoAndReturn(func(ctx context.Context, req *workflowservice.StartWorkflowExecutionRequest, _ ...grpc.CallOption) (*workflowservice.StartWorkflowExecutionResponse, error) { - countAction() - rpcMu.Lock() - startRequests[req.GetRequestId()] = struct{}{} - rpcMu.Unlock() - if startCalls != nil { - call := &StartCall{Request: common.CloneProto(req), t: t, result: make(chan startResponse, 1), ledger: external} - select { - case startCalls <- call: - case <-ctx.Done(): - return nil, ctx.Err() - } - select { - case result := <-call.result: - return result.response, result.err - case <-ctx.Done(): - return nil, ctx.Err() - } - } - rpcMu.Lock() - rejected, lostReply := faults.rejected > 0, faults.rejected == 0 && faults.lostReply > 0 - if rejected { - faults.rejected-- - } else if lostReply { - faults.lostReply-- - } - rpcMu.Unlock() - if rejected { - return nil, serviceerror.NewUnavailable("start rejected by driver") - } - accepted, err := external.accept(req) - if err != nil { - return nil, err - } - if lostReply { - return nil, serviceerror.NewUnavailable("start accepted, response lost by driver") - } - return &workflowservice.StartWorkflowExecutionResponse{RunId: accepted.RunID}, nil - }).AnyTimes() - history.EXPECT().TerminateWorkflowExecution(gomock.Any(), gomock.Any()). - DoAndReturn(func(context.Context, *historyservice.TerminateWorkflowExecutionRequest, ...grpc.CallOption) (*historyservice.TerminateWorkflowExecutionResponse, error) { - countAction() - return &historyservice.TerminateWorkflowExecutionResponse{}, nil - }).AnyTimes() - history.EXPECT().RequestCancelWorkflowExecution(gomock.Any(), gomock.Any()). - DoAndReturn(func(context.Context, *historyservice.RequestCancelWorkflowExecutionRequest, ...grpc.CallOption) (*historyservice.RequestCancelWorkflowExecutionResponse, error) { - countAction() - return &historyservice.RequestCancelWorkflowExecutionResponse{}, nil - }).AnyTimes() - // The callbacks task (used for migrated running workflows) describes each - // running workflow; report it completed so watchers resolve deterministically. - history.EXPECT().DescribeWorkflowExecution(gomock.Any(), gomock.Any()). - DoAndReturn(func(_ context.Context, _ *historyservice.DescribeWorkflowExecutionRequest, _ ...grpc.CallOption) (*historyservice.DescribeWorkflowExecutionResponse, error) { - return &historyservice.DescribeWorkflowExecutionResponse{ - WorkflowExecutionInfo: &workflowpb.WorkflowExecutionInfo{ - Status: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, - CloseTime: timestamppb.New(ts.Now()), - }, - }, nil - }).AnyTimes() - + services := newFakeServices(t, gomock.NewController(t), ts, cfg.controlledStarts) metricsHandler := metricstest.NewCaptureHandler() - capture := metricsHandler.StartCapture() + engineOpts := []Option{ - WithLogger(logger), + WithLogger(testlogger.NewTestLogger(t, testlogger.FailOnExpectedErrorOnly)), WithSchedulerConfig(cfg.config), WithTimeSource(ts), WithInvokerTimeSource(ts), WithInvokerMetrics(metricsHandler), - WithClients(frontend, history), + WithClients(services.Frontend, services.History), } if cfg.noStateCheck { engineOpts = append(engineOpts, WithoutStateCheck()) } - base := NewEngine(t, engineOpts...) - engine := base.Engine - registry := base.Registry d := &Driver{ - base: base, - recordCalls: make(chan *RecordCall), - external: external, startCalls: startCalls, recorded: make(map[string]ExternalStart), - replacingSeen: make(map[string]struct{}), - T: t, - Engine: engine, - Registry: registry, - TimeSource: ts, - Logger: logger, - Frontend: frontend, - History: history, - Config: cfg.config, - actionRPCs: actionRPCs, - startRequests: startRequests, - rpcMu: rpcMu, - faults: faults, - metricsCapture: capture, - maxRunDuration: cfg.maxRun, - AfterStep: cfg.afterStep, - execKey: chasm.ExecutionKey{NamespaceID: NamespaceID, BusinessID: ScheduleID}, + SchedulerEngine: NewEngine(t, engineOpts...), + fakeServices: services, + recordCalls: make(chan *RecordCall), + recorded: make(map[string]ExternalStart), + replacingSeen: make(map[string]struct{}), + metricsCapture: metricsHandler.StartCapture(), + maxRunDuration: cfg.maxRun, + AfterStep: cfg.afterStep, } - d.ref = chasm.NewComponentRef[*scheduler.Scheduler](d.execKey) d.runner = d.newTaskRunner(cfg.maxLag) return d, cfg } -// startExecution starts the scheduler execution using the given start function. -func (d *Driver) startExecution(startFn func(chasm.MutableContext) (*scheduler.Scheduler, error)) { +// start starts the scheduler execution using the given start function. +func (d *Driver) start(startFn func(chasm.MutableContext) (*scheduler.Scheduler, error)) { d.T.Helper() - - engineCtx := chasm.NewEngineContext(context.Background(), d.Engine) - - _, err := chasm.StartExecution(engineCtx, d.execKey, func(mctx chasm.MutableContext, _ *struct{}) (*scheduler.Scheduler, error) { - return startFn(mctx) - }, (*struct{})(nil), chasm.WithRequestID("create-req")) - if err != nil { + if err := d.Start(startFn); err != nil { d.T.Fatalf("failed to start scheduler execution: %v", err) } - d.ReadScheduler(func(s *scheduler.Scheduler, _ chasm.Context) { d.initialActionCount = s.Info.GetActionCount() }) } -// StateViolations evaluates the shared stuck and idle-deadline invariants -// against the last committed state without failing the test. See -// [WithoutStateInvariant]. -func (d *Driver) StateViolations() []string { - d.T.Helper() - return d.base.StateViolations() -} - // Now returns the engine's current clock time. func (d *Driver) Now() time.Time { return d.TimeSource.Now() @@ -381,12 +220,7 @@ func (d *Driver) Now() time.Time { // ReadScheduler reads the scheduler root component and invokes fn with it. func (d *Driver) ReadScheduler(fn func(s *scheduler.Scheduler, ctx chasm.Context)) { d.T.Helper() - engineCtx := chasm.NewEngineContext(context.Background(), d.Engine) - err := d.Engine.ReadComponent(engineCtx, d.ref, func(ctx chasm.Context, c chasm.Component) error { - s, ok := c.(*scheduler.Scheduler) - if !ok { - d.T.Fatalf("root component is %T, want *scheduler.Scheduler", c) - } + err := d.Read(func(s *scheduler.Scheduler, ctx chasm.Context) error { fn(s, ctx) return nil }) @@ -420,10 +254,26 @@ func (d *Driver) StepSideEffect() bool { return d.runner.StepSideEffect() } +// RunToQuiescence steps until no task is pending or maxSteps is reached. It +// returns the number of steps taken and whether the schedule reached quiescence +// (no pending task). A normal running schedule never quiesces and will stop at +// maxSteps with quiescent == false. +func (d *Driver) RunToQuiescence(maxSteps int) (steps int, quiescent bool) { + d.T.Helper() + return d.runner.RunToQuiescence(maxSteps) +} + +// HasPendingTask reports whether any task (pure or side-effect) is currently +// scheduled to fire. +func (d *Driver) HasPendingTask() bool { + d.T.Helper() + return d.runner.HasPendingTask() +} + // newTaskRunner wires the scheduler-specific behaviour into a generic task // runner. func (d *Driver) newTaskRunner(maxLag time.Duration) *chasmtest.TaskRunner { - return chasmtest.NewTaskRunner(d.T, d.Engine, d.ref, d.TimeSource, + return chasmtest.NewTaskRunner(d.T, d.Engine, d.Ref, d.TimeSource, chasmtest.WithMaxSideEffectLag(maxLag), chasmtest.WithAsyncEngine(func(engine chasm.Engine) chasm.Engine { return &recordingEngine{Engine: engine, calls: d.recordCalls} @@ -438,15 +288,17 @@ func (d *Driver) newTaskRunner(maxLag time.Duration) *chasmtest.TaskRunner { }, AfterSideEffect: func() { d.maxActionsObserved = max(d.maxActionsObserved, d.actionCount()-d.actionsBefore) }, AfterSideEffectCommit: d.observeRecordedStarts, - AfterStep: func() { - if d.AfterStep != nil { - d.AfterStep(d) - } - }, + AfterStep: d.runAfterStep, }), ) } +func (d *Driver) runAfterStep() { + if d.AfterStep != nil { + d.AfterStep(d) + } +} + // completeWorkflowsBefore completes any workflow that would exceed // maxRunDuration before the clock moves to t, moving the clock to the deadline // first. @@ -479,42 +331,61 @@ func (d *Driver) oldestRunningStartTime() (time.Time, bool) { if s.Closed || s.IsSentinel() { return } - for _, start := range s.Invoker.Get(ctx).GetBufferedStarts() { - if start.GetRunId() == "" || start.GetCompleted() != nil { - continue - } - if st := start.GetStartTime().AsTime(); !have || st.Before(oldest) { - oldest, have = st, true - } + if start := oldestRunning(s.Invoker.Get(ctx).GetBufferedStarts()); start != nil { + oldest, have = start.GetStartTime().AsTime(), true } }) return oldest, have } +// oldestRunning returns the earliest-started buffered start that has a run but +// has not completed, or nil. +func oldestRunning(starts []*schedulespb.BufferedStart) *schedulespb.BufferedStart { + var oldest *schedulespb.BufferedStart + for _, start := range starts { + if start.GetRunId() == "" || start.GetCompleted() != nil { + continue + } + if oldest == nil || start.GetStartTime().AsTime().Before(oldest.GetStartTime().AsTime()) { + oldest = start + } + } + return oldest +} + +// StateViolations evaluates the shared stuck and idle-deadline invariants +// against the last committed state without failing the test. See +// [WithoutStateInvariant]. +func (d *Driver) StateViolations() []string { + d.T.Helper() + return d.SchedulerEngine.StateViolations() +} + +// apply runs op against the scheduler root in a single transaction. On success +// the AfterStep hook runs, so invariants are checked at the new settled state. +// It returns the operation's error without failing the test, so callers driving +// random operations can tolerate rejected ones (e.g. ErrClosed once the +// schedule has closed). +func (d *Driver) apply(op func(*scheduler.Scheduler, chasm.MutableContext) error) error { + d.T.Helper() + err := d.SchedulerEngine.Update(op) + if err == nil { + d.runAfterStep() + } + return err +} + // CompleteWorkflow delivers a successful completion callback for the oldest // running workflow, through Scheduler.HandleNexusCompletion. It returns false // if no workflow is running. func (d *Driver) CompleteWorkflow() bool { d.T.Helper() completed := false - engineCtx := chasm.NewEngineContext(context.Background(), d.Engine) - _, err := d.Engine.UpdateComponent(engineCtx, d.ref, func(mctx chasm.MutableContext, c chasm.Component) error { - s, ok := c.(*scheduler.Scheduler) - if !ok { - return fmt.Errorf("root component is %T, want *scheduler.Scheduler", c) - } + err := d.SchedulerEngine.Update(func(s *scheduler.Scheduler, mctx chasm.MutableContext) error { if s.Closed || s.IsSentinel() { return nil } - var oldest *schedulespb.BufferedStart - for _, start := range s.Invoker.Get(mctx).GetBufferedStarts() { - if start.GetRunId() == "" || start.GetCompleted() != nil { - continue - } - if oldest == nil || start.GetStartTime().AsTime().Before(oldest.GetStartTime().AsTime()) { - oldest = start - } - } + oldest := oldestRunning(s.Invoker.Get(mctx).GetBufferedStarts()) if oldest == nil { return nil } @@ -528,42 +399,32 @@ func (d *Driver) CompleteWorkflow() bool { if err != nil { d.T.Fatalf("CompleteWorkflow failed: %v", err) } - if completed && d.AfterStep != nil { - d.AfterStep(d) + if completed { + d.runAfterStep() } return completed } -// RunToQuiescence steps until no task is pending or maxSteps is reached. It -// returns the number of steps taken and whether the schedule reached quiescence -// (no pending task). A normal running schedule never quiesces and will stop at -// maxSteps with quiescent == false. -func (d *Driver) RunToQuiescence(maxSteps int) (steps int, quiescent bool) { - d.T.Helper() - return d.runner.RunToQuiescence(maxSteps) -} - -// HasPendingTask reports whether any task (pure or side-effect) is currently -// scheduled to fire. -func (d *Driver) HasPendingTask() bool { - d.T.Helper() - return d.runner.HasPendingTask() +// DeliverCompletion delivers a completion callback for requestID. It also +// supports accepted starts whose RPC results have not been recorded locally, and +// duplicate or obsolete callback request identities. +func (d *Driver) DeliverCompletion(requestID string) error { + return d.SchedulerEngine.Update(func(s *scheduler.Scheduler, mctx chasm.MutableContext) error { + return s.HandleNexusCompletion(mctx, &persistencespb.ChasmNexusCompletion{ + RequestId: requestID, CloseTime: timestamppb.New(d.Now()), + Outcome: &persistencespb.ChasmNexusCompletion_Success{Success: &commonpb.Payload{}}, + }) + }) } // Patch applies a SchedulePatch through the production Scheduler.Patch path in a // single transaction. It returns the scheduler's error (e.g. ErrClosed when the -// schedule has already closed) without failing the test, so callers driving -// random operations can tolerate rejected patches. On success the AfterStep hook -// runs so invariants are checked at the new settled state. +// schedule has already closed) without failing the test. On success the +// AfterStep hook runs. func (d *Driver) Patch(patch *schedulepb.SchedulePatch) error { d.T.Helper() - engineCtx := chasm.NewEngineContext(context.Background(), d.Engine) - _, err := d.Engine.UpdateComponent(engineCtx, d.ref, func(mctx chasm.MutableContext, c chasm.Component) error { - s, ok := c.(*scheduler.Scheduler) - if !ok { - return fmt.Errorf("root component is %T, want *scheduler.Scheduler", c) - } - _, perr := s.Patch(mctx, &schedulerpb.PatchScheduleRequest{ + return d.apply(func(s *scheduler.Scheduler, mctx chasm.MutableContext) error { + _, err := s.Patch(mctx, &schedulerpb.PatchScheduleRequest{ NamespaceId: NamespaceID, FrontendRequest: &workflowservice.PatchScheduleRequest{ Namespace: Namespace, @@ -571,53 +432,21 @@ func (d *Driver) Patch(patch *schedulepb.SchedulePatch) error { Patch: patch, }, }) - return perr + return err }) - if err == nil && d.AfterStep != nil { - d.AfterStep(d) - } - return err -} - -// FailNextStarts makes the next n StartWorkflowExecution RPCs fail before the -// start is accepted, then lets later RPCs (the scheduler's retries) succeed. -// It does not apply under WithControlledStarts, where the test decides each -// RPC's outcome. -func (d *Driver) FailNextStarts(n int) { - d.rpcMu.Lock() - defer d.rpcMu.Unlock() - d.faults.rejected += n -} - -// LoseNextStartResponses makes the next n StartWorkflowExecution RPCs (after -// any FailNextStarts) accept the start but return an error, so the scheduler -// must retry with the same request ID and rely on deduplication. -func (d *Driver) LoseNextStartResponses(n int) { - d.rpcMu.Lock() - defer d.rpcMu.Unlock() - d.faults.lostReply += n -} - -// Backfill requests a backfill of the given range via Patch. -func (d *Driver) Backfill(req *schedulepb.BackfillRequest) error { - return d.Patch(&schedulepb.SchedulePatch{BackfillRequest: []*schedulepb.BackfillRequest{req}}) } // Update applies mutate to a copy of the current schedule definition and // replaces the schedule with it through the production Scheduler.Update path, // without a conflict token. It returns the scheduler's error (e.g. ErrClosed) -// without failing the test. On success the AfterStep hook runs. +// without failing the test. On success the AfterStep hook runs. (The embedded +// SchedulerEngine.Update is the lower-level typed helper.) func (d *Driver) Update(mutate func(*schedulepb.Schedule)) error { d.T.Helper() - engineCtx := chasm.NewEngineContext(context.Background(), d.Engine) - _, err := d.Engine.UpdateComponent(engineCtx, d.ref, func(mctx chasm.MutableContext, c chasm.Component) error { - s, ok := c.(*scheduler.Scheduler) - if !ok { - return fmt.Errorf("root component is %T, want *scheduler.Scheduler", c) - } + return d.apply(func(s *scheduler.Scheduler, mctx chasm.MutableContext) error { next := common.CloneProto(s.Schedule) mutate(next) - _, uerr := s.Update(mctx, &schedulerpb.UpdateScheduleRequest{ + _, err := s.Update(mctx, &schedulerpb.UpdateScheduleRequest{ NamespaceId: NamespaceID, FrontendRequest: &workflowservice.UpdateScheduleRequest{ Namespace: Namespace, @@ -625,12 +454,13 @@ func (d *Driver) Update(mutate func(*schedulepb.Schedule)) error { Schedule: next, }, }) - return uerr + return err }) - if err == nil && d.AfterStep != nil { - d.AfterStep(d) - } - return err +} + +// Backfill requests a backfill of the given range via Patch. +func (d *Driver) Backfill(req *schedulepb.BackfillRequest) error { + return d.Patch(&schedulepb.SchedulePatch{BackfillRequest: []*schedulepb.BackfillRequest{req}}) } // Pause pauses the schedule via Patch. @@ -653,191 +483,15 @@ func (d *Driver) TriggerImmediately() error { // MigrateToWorkflow initiates a V2->V1 migration: it pauses the schedule and // schedules the SchedulerMigrateToWorkflowTask side-effect task. Step the driver // afterward to fire the task (which calls the history client's -// StartWorkflowExecution with the exported StartScheduleArgs). +// StartWorkflowExecution with the exported StartScheduleArgs). It does not run +// the AfterStep hook. func (d *Driver) MigrateToWorkflow() error { d.T.Helper() - engineCtx := chasm.NewEngineContext(context.Background(), d.Engine) - _, err := d.Engine.UpdateComponent(engineCtx, d.ref, func(mctx chasm.MutableContext, c chasm.Component) error { - s, ok := c.(*scheduler.Scheduler) - if !ok { - return fmt.Errorf("root component is %T, want *scheduler.Scheduler", c) - } - _, merr := s.MigrateToWorkflow(mctx, &schedulerpb.MigrateToWorkflowRequest{ + return d.SchedulerEngine.Update(func(s *scheduler.Scheduler, mctx chasm.MutableContext) error { + _, err := s.MigrateToWorkflow(mctx, &schedulerpb.MigrateToWorkflowRequest{ NamespaceId: NamespaceID, ScheduleId: ScheduleID, }) - return merr + return err }) - return err -} - -// Snapshot captures the observable scheduler state at a settled point (after a -// step's CloseTransaction). It is the input to invariant checks. -type Snapshot struct { - UntransferableActions int - AcceptedStarts map[string]ExternalStart - RecordedStarts map[string]ExternalStart - ActionCount int64 - InitialActionCount int64 - - // Now is the engine clock at capture time. - Now time.Time - // HasPendingTask is true if any task is scheduled to fire. - HasPendingTask bool - - Closed bool - IsSentinel bool - Paused bool - // BackfillerCount is the number of live backfiller components. - BackfillerCount int - // IsHeldOpen mirrors Scheduler.isHeldOpen: the schedule must stay open - // regardless of having no work (paused or pending backfill, non-sentinel). - IsHeldOpen bool - // IdleCloseTime is the armed idle-close deadline, or zero if no idle task is armed. - IdleCloseTime time.Time - - // GeneratorLPT / InvokerLPT are the generator and invoker high-water marks. - GeneratorLPT time.Time - InvokerLPT time.Time - - // BufferedStartsCount is len(invoker.GetBufferedStarts()) — the total buffered - // starts (pending + retained). MaxBufferSize is the configured cap; the - // buffer-bound invariant requires BufferedStartsCount <= MaxBufferSize. - BufferedStartsCount int - MaxBufferSize int - - // MaxActionsObserved is the high-water mark of action RPCs (start + terminate + - // cancel) taken by any single InvokerExecuteTask so far. MaxActionsPerExecution - // is the configured per-execution cap; the action-budget invariant requires - // MaxActionsObserved <= MaxActionsPerExecution. - MaxActionsObserved int - MaxActionsPerExecution int - - // Start ledger inputs. PendingStarts are buffered starts with no RunId yet. - // StartRequested holds every RequestId sent to StartWorkflowExecution. - // OverlapSkipped and DroppedStarts count the starts the scheduler discarded - // on purpose (Info.OverlapSkipped, and the ScheduleBufferedStartDropped - // metric for missed-catchup and paused/limited drops). - PendingStarts map[string]struct{} - StartRequested map[string]struct{} - // ReplacingStarts counts the pending starts that no start RPC has been attempted for (including deferred ones) whose overlap policy - // is CANCEL_OTHER or TERMINATE_OTHER. With nothing running, such a start - // replaces the one the buffer would otherwise have started - // (legacyscheduler.ProcessBuffer), discarding it without counting an overlap - // skip, so each can account for one silent disappearance. ReplacingSeen is - // the number of distinct such starts ever seen in the buffer in any state, - // which covers a replacer that was buffered and started between two snapshots. - ReplacingStarts int - ReplacingSeen int - OverlapSkipped int64 - DroppedStarts int64 - // Migrating is true once the schedule hands off to a V1 workflow, which - // takes its buffered starts with it. - Migrating bool - // DeferredStarts are pending starts waiting on a workflow completion - // (Attempt == -1); RunningWorkflows counts started, uncompleted workflows. - DeferredStarts int - RunningWorkflows int -} - -// Snapshot reads current scheduler state into a Snapshot. -func (d *Driver) Snapshot() Snapshot { - d.T.Helper() - d.observeRecordedStarts() - tweakables := d.Config.Tweakables(Namespace) - snap := Snapshot{ - AcceptedStarts: d.external.snapshot(), RecordedStarts: maps.Clone(d.recorded), - InitialActionCount: d.initialActionCount, - Now: d.Now(), - HasPendingTask: d.HasPendingTask(), - MaxBufferSize: tweakables.MaxBufferSize, - MaxActionsObserved: d.maxActionsObserved, - MaxActionsPerExecution: tweakables.MaxActionsPerExecution, - PendingStarts: make(map[string]struct{}), - StartRequested: d.startRequestsCopy(), - DroppedStarts: d.droppedStarts(), - } - d.ReadScheduler(func(s *scheduler.Scheduler, ctx chasm.Context) { - snap.ActionCount = s.Info.GetActionCount() - snap.Closed = s.Closed - snap.IsSentinel = s.IsSentinel() - snap.Paused = s.Schedule.GetState().GetPaused() - snap.BackfillerCount = len(s.Backfillers) - snap.OverlapSkipped = s.Info.GetOverlapSkipped() - snap.Migrating = s.GetWorkflowMigration() != nil - // Mirror Scheduler.isHeldOpen() from public state. - snap.IsHeldOpen = !snap.IsSentinel && (snap.Paused || snap.BackfillerCount > 0) - if s.IdleCloseTime != nil { - snap.IdleCloseTime = s.IdleCloseTime.AsTime() - } - // Sentinels have no generator/invoker sub-components. - if !snap.IsSentinel { - if g := s.Generator.Get(ctx); g != nil && g.LastProcessedTime != nil { - snap.GeneratorLPT = g.LastProcessedTime.AsTime() - } - if inv := s.Invoker.Get(ctx); inv != nil { - snap.UntransferableActions = len(inv.GetCancelWorkflows()) + len(inv.GetTerminateWorkflows()) - snap.BufferedStartsCount = len(inv.GetBufferedStarts()) - snap.addBufferedStarts(inv.GetBufferedStarts(), s.Schedule.GetPolicies().GetOverlapPolicy(), d.replacingSeen) - snap.ReplacingSeen = len(d.replacingSeen) - if inv.LastProcessedTime != nil { - snap.InvokerLPT = inv.LastProcessedTime.AsTime() - } - } - } - }) - return snap -} - -// addBufferedStarts classifies buffered starts into pending (deferred or not) -// and running. scheduleOverlap resolves starts with an unspecified policy. -// replacingSeen accumulates the request IDs of replacing starts across snapshots. -func (snap *Snapshot) addBufferedStarts(starts []*schedulespb.BufferedStart, scheduleOverlap enumspb.ScheduleOverlapPolicy, replacingSeen map[string]struct{}) { - for _, start := range starts { - policy := cmp.Or(start.GetOverlapPolicy(), scheduleOverlap) - replacing := policy == enumspb.SCHEDULE_OVERLAP_POLICY_CANCEL_OTHER || - policy == enumspb.SCHEDULE_OVERLAP_POLICY_TERMINATE_OTHER - if replacing { - replacingSeen[start.GetRequestId()] = struct{}{} - } - switch { - case start.GetRunId() == "": - if replacing && start.GetAttempt() <= 0 { - snap.ReplacingStarts++ - } - if start.GetAttempt() > 0 { - snap.UntransferableActions++ - } - snap.PendingStarts[start.GetRequestId()] = struct{}{} - if start.GetAttempt() == -1 { - snap.DeferredStarts++ - } - case start.GetCompleted() == nil: - snap.RunningWorkflows++ - default: - } - } -} - -// droppedStarts sums the ScheduleBufferedStartDropped counter. -func (d *Driver) droppedStarts() int64 { - var total int64 - for _, rec := range d.metricsCapture.SnapshotMetric(metrics.ScheduleBufferedStartDropped.Name()) { - if v, ok := rec.Value.(int64); ok { - total += v - } - } - return total -} - -func (d *Driver) actionCount() int { - d.rpcMu.Lock() - defer d.rpcMu.Unlock() - return *d.actionRPCs -} - -func (d *Driver) startRequestsCopy() map[string]struct{} { - d.rpcMu.Lock() - defer d.rpcMu.Unlock() - return maps.Clone(d.startRequests) } diff --git a/chasm/lib/scheduler/schedulertest/external.go b/chasm/lib/scheduler/schedulertest/external.go index d84907ab65c..9f33968b4cc 100644 --- a/chasm/lib/scheduler/schedulertest/external.go +++ b/chasm/lib/scheduler/schedulertest/external.go @@ -6,13 +6,10 @@ import ( "maps" "sync" - commonpb "go.temporal.io/api/common/v1" "go.temporal.io/api/workflowservice/v1" - persistencespb "go.temporal.io/server/api/persistence/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/chasmtest" "go.temporal.io/server/chasm/lib/scheduler" - "google.golang.org/protobuf/types/known/timestamppb" ) // ExternalStart records acceptance outside the scheduler's transaction. @@ -179,19 +176,6 @@ func (c *RecordCall) Record() error { return err } -// DeliverCompletion also supports accepted starts whose RPC results have not -// been recorded locally, and duplicate or obsolete callback request identities. -func (d *Driver) DeliverCompletion(requestID string) error { - _, err := d.Engine.UpdateComponent(chasm.NewEngineContext(context.Background(), d.Engine), d.ref, - func(ctx chasm.MutableContext, c chasm.Component) error { - return c.(*scheduler.Scheduler).HandleNexusCompletion(ctx, &persistencespb.ChasmNexusCompletion{ - RequestId: requestID, CloseTime: timestamppb.New(d.Now()), - Outcome: &persistencespb.ChasmNexusCompletion_Success{Success: &commonpb.Payload{}}, - }) - }) - return err -} - func (d *Driver) observeRecordedStarts() { accepted := d.external.snapshot() d.ReadScheduler(func(s *scheduler.Scheduler, ctx chasm.Context) { diff --git a/chasm/lib/scheduler/schedulertest/fakes.go b/chasm/lib/scheduler/schedulertest/fakes.go new file mode 100644 index 00000000000..51227508fe6 --- /dev/null +++ b/chasm/lib/scheduler/schedulertest/fakes.go @@ -0,0 +1,173 @@ +package schedulertest + +import ( + "context" + "maps" + "sync" + + enumspb "go.temporal.io/api/enums/v1" + "go.temporal.io/api/serviceerror" + workflowpb "go.temporal.io/api/workflow/v1" + "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/server/api/historyservice/v1" + "go.temporal.io/server/api/historyservicemock/v1" + "go.temporal.io/server/chasm/chasmtest" + "go.temporal.io/server/common" + "go.temporal.io/server/common/clock" + "go.temporal.io/server/common/testing/mockapi/workflowservicemock/v1" + "go.uber.org/mock/gomock" + "google.golang.org/grpc" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// startFaults counts the upcoming StartWorkflowExecution RPCs that fail. +type startFaults struct { + rejected int // the RPC fails without the start being accepted + lostReply int // the start is accepted but the response is lost +} + +// fakeServices is the world outside the scheduler: a frontend that accepts +// starts into an external ledger, a history service that accepts terminate and +// cancel requests, and the counters and fault injection the invariants read. +type fakeServices struct { + Frontend *workflowservicemock.MockWorkflowServiceClient + History *historyservicemock.MockHistoryServiceClient + + // external records the starts the fake frontend accepted, outside the + // scheduler's transactions. + external *externalLedger + // startCalls is non-nil under WithControlledStarts, where a test decides + // each start RPC's outcome. + startCalls chan *StartCall + + // rpcMu guards the fields below: InvokerExecuteTask issues starts from + // concurrent goroutines. + rpcMu sync.Mutex + // actionRPCs counts every action RPC (StartWorkflowExecution, + // TerminateWorkflowExecution, RequestCancelWorkflowExecution), so a step can + // measure how many actions a single ExecuteTask took. + actionRPCs int + // startRequests records the RequestId of every StartWorkflowExecution RPC. + startRequests map[string]struct{} + // faults injects failures into uncontrolled start RPCs. + faults startFaults +} + +// newFakeServices wires happy-path stubs; tests can layer more specific +// expectations on Frontend and History. +func newFakeServices(t chasmtest.TB, ctrl *gomock.Controller, ts clock.TimeSource, controlledStarts bool) *fakeServices { + f := &fakeServices{ + Frontend: workflowservicemock.NewMockWorkflowServiceClient(ctrl), + History: historyservicemock.NewMockHistoryServiceClient(ctrl), + external: &externalLedger{starts: make(map[string]ExternalStart)}, + startRequests: make(map[string]struct{}), + } + if controlledStarts { + f.startCalls = make(chan *StartCall) + } + + f.Frontend.EXPECT().StartWorkflowExecution(gomock.Any(), gomock.Any()). + DoAndReturn(func(ctx context.Context, req *workflowservice.StartWorkflowExecutionRequest, _ ...grpc.CallOption) (*workflowservice.StartWorkflowExecutionResponse, error) { + f.rpcMu.Lock() + f.actionRPCs++ + f.startRequests[req.GetRequestId()] = struct{}{} + f.rpcMu.Unlock() + if f.startCalls != nil { + return f.controlledStart(ctx, t, req) + } + f.rpcMu.Lock() + rejected, lostReply := f.faults.rejected > 0, f.faults.rejected == 0 && f.faults.lostReply > 0 + if rejected { + f.faults.rejected-- + } else if lostReply { + f.faults.lostReply-- + } + f.rpcMu.Unlock() + if rejected { + return nil, serviceerror.NewUnavailable("start rejected by driver") + } + accepted, err := f.external.accept(req) + if err != nil { + return nil, err + } + if lostReply { + return nil, serviceerror.NewUnavailable("start accepted, response lost by driver") + } + return &workflowservice.StartWorkflowExecutionResponse{RunId: accepted.RunID}, nil + }).AnyTimes() + f.History.EXPECT().TerminateWorkflowExecution(gomock.Any(), gomock.Any()). + DoAndReturn(func(context.Context, *historyservice.TerminateWorkflowExecutionRequest, ...grpc.CallOption) (*historyservice.TerminateWorkflowExecutionResponse, error) { + f.countAction() + return &historyservice.TerminateWorkflowExecutionResponse{}, nil + }).AnyTimes() + f.History.EXPECT().RequestCancelWorkflowExecution(gomock.Any(), gomock.Any()). + DoAndReturn(func(context.Context, *historyservice.RequestCancelWorkflowExecutionRequest, ...grpc.CallOption) (*historyservice.RequestCancelWorkflowExecutionResponse, error) { + f.countAction() + return &historyservice.RequestCancelWorkflowExecutionResponse{}, nil + }).AnyTimes() + // The callbacks task (used for migrated running workflows) describes each + // running workflow; report it completed so watchers resolve deterministically. + f.History.EXPECT().DescribeWorkflowExecution(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, _ *historyservice.DescribeWorkflowExecutionRequest, _ ...grpc.CallOption) (*historyservice.DescribeWorkflowExecutionResponse, error) { + return &historyservice.DescribeWorkflowExecutionResponse{ + WorkflowExecutionInfo: &workflowpb.WorkflowExecutionInfo{ + Status: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, + CloseTime: timestamppb.New(ts.Now()), + }, + }, nil + }).AnyTimes() + return f +} + +// controlledStart hands the RPC to the test and waits for its Respond. +func (f *fakeServices) controlledStart(ctx context.Context, t chasmtest.TB, req *workflowservice.StartWorkflowExecutionRequest) (*workflowservice.StartWorkflowExecutionResponse, error) { + call := &StartCall{Request: common.CloneProto(req), t: t, result: make(chan startResponse, 1), ledger: f.external} + select { + case f.startCalls <- call: + case <-ctx.Done(): + return nil, ctx.Err() + } + select { + case result := <-call.result: + return result.response, result.err + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +func (f *fakeServices) countAction() { + f.rpcMu.Lock() + defer f.rpcMu.Unlock() + f.actionRPCs++ +} + +func (f *fakeServices) actionCount() int { + f.rpcMu.Lock() + defer f.rpcMu.Unlock() + return f.actionRPCs +} + +func (f *fakeServices) startRequestsCopy() map[string]struct{} { + f.rpcMu.Lock() + defer f.rpcMu.Unlock() + return maps.Clone(f.startRequests) +} + +// FailNextStarts makes the next n StartWorkflowExecution RPCs fail before the +// start is accepted, then lets later RPCs (the scheduler's retries) succeed. +// It does not apply under WithControlledStarts, where the test decides each +// RPC's outcome. +func (f *fakeServices) FailNextStarts(n int) { + f.rpcMu.Lock() + defer f.rpcMu.Unlock() + f.faults.rejected += n +} + +// LoseNextStartResponses makes the next n StartWorkflowExecution RPCs (after +// any FailNextStarts) accept the start but return an error, so the scheduler +// must retry with the same request ID and rely on deduplication. +func (f *fakeServices) LoseNextStartResponses(n int) { + f.rpcMu.Lock() + defer f.rpcMu.Unlock() + f.faults.lostReply += n +} diff --git a/chasm/lib/scheduler/schedulertest/inflight_test.go b/chasm/lib/scheduler/schedulertest/inflight_test.go index 5841ef88bf9..3e180941121 100644 --- a/chasm/lib/scheduler/schedulertest/inflight_test.go +++ b/chasm/lib/scheduler/schedulertest/inflight_test.go @@ -25,7 +25,7 @@ func TestStartResultSurvivesBufferReordering(t *testing.T) { require.NoError(t, err) acceptedSecond, err := second.Accept() require.NoError(t, err) - _, err = d.Engine.UpdateComponent(chasm.NewEngineContext(context.Background(), d.Engine), d.ref, + _, err = d.Engine.UpdateComponent(chasm.NewEngineContext(context.Background(), d.Engine), d.Ref, func(ctx chasm.MutableContext, c chasm.Component) error { starts := c.(*scheduler.Scheduler).Invoker.Get(ctx).BufferedStarts starts[0], starts[1] = starts[1], starts[0] diff --git a/chasm/lib/scheduler/schedulertest/snapshot.go b/chasm/lib/scheduler/schedulertest/snapshot.go new file mode 100644 index 00000000000..11a17d449d4 --- /dev/null +++ b/chasm/lib/scheduler/schedulertest/snapshot.go @@ -0,0 +1,172 @@ +package schedulertest + +import ( + "cmp" + "maps" + "time" + + enumspb "go.temporal.io/api/enums/v1" + schedulespb "go.temporal.io/server/api/schedule/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/scheduler" + "go.temporal.io/server/common/metrics" +) + +// Snapshot captures the observable scheduler state at a settled point (after a +// step's CloseTransaction). It is the input to invariant checks. +type Snapshot struct { + UntransferableActions int + AcceptedStarts map[string]ExternalStart + RecordedStarts map[string]ExternalStart + ActionCount int64 + InitialActionCount int64 + + // Now is the engine clock at capture time. + Now time.Time + // HasPendingTask is true if any task is scheduled to fire. + HasPendingTask bool + + Closed bool + IsSentinel bool + Paused bool + // BackfillerCount is the number of live backfiller components. + BackfillerCount int + // IsHeldOpen mirrors Scheduler.isHeldOpen: the schedule must stay open + // regardless of having no work (paused or pending backfill, non-sentinel). + IsHeldOpen bool + // IdleCloseTime is the armed idle-close deadline, or zero if no idle task is armed. + IdleCloseTime time.Time + + // GeneratorLPT / InvokerLPT are the generator and invoker high-water marks. + GeneratorLPT time.Time + InvokerLPT time.Time + + // BufferedStartsCount is len(invoker.GetBufferedStarts()) — the total buffered + // starts (pending + retained). MaxBufferSize is the configured cap; the + // buffer-bound invariant requires BufferedStartsCount <= MaxBufferSize. + BufferedStartsCount int + MaxBufferSize int + + // MaxActionsObserved is the high-water mark of action RPCs (start + terminate + + // cancel) taken by any single InvokerExecuteTask so far. MaxActionsPerExecution + // is the configured per-execution cap; the action-budget invariant requires + // MaxActionsObserved <= MaxActionsPerExecution. + MaxActionsObserved int + MaxActionsPerExecution int + + // Start ledger inputs. PendingStarts are buffered starts with no RunId yet. + // StartRequested holds every RequestId sent to StartWorkflowExecution. + // OverlapSkipped and DroppedStarts count the starts the scheduler discarded + // on purpose (Info.OverlapSkipped, and the ScheduleBufferedStartDropped + // metric for missed-catchup and paused/limited drops). + PendingStarts map[string]struct{} + StartRequested map[string]struct{} + // ReplacingStarts counts the pending starts that no start RPC has been attempted for (including deferred ones) whose overlap policy + // is CANCEL_OTHER or TERMINATE_OTHER. With nothing running, such a start + // replaces the one the buffer would otherwise have started + // (legacyscheduler.ProcessBuffer), discarding it without counting an overlap + // skip, so each can account for one silent disappearance. ReplacingSeen is + // the number of distinct such starts ever seen in the buffer in any state, + // which covers a replacer that was buffered and started between two snapshots. + ReplacingStarts int + ReplacingSeen int + OverlapSkipped int64 + DroppedStarts int64 + // Migrating is true once the schedule hands off to a V1 workflow, which + // takes its buffered starts with it. + Migrating bool + // DeferredStarts are pending starts waiting on a workflow completion + // (Attempt == -1); RunningWorkflows counts started, uncompleted workflows. + DeferredStarts int + RunningWorkflows int +} + +// Snapshot reads current scheduler state into a Snapshot. +func (d *Driver) Snapshot() Snapshot { + d.T.Helper() + d.observeRecordedStarts() + tweakables := d.Config.Tweakables(Namespace) + snap := Snapshot{ + AcceptedStarts: d.external.snapshot(), RecordedStarts: maps.Clone(d.recorded), + InitialActionCount: d.initialActionCount, + Now: d.Now(), + HasPendingTask: d.HasPendingTask(), + MaxBufferSize: tweakables.MaxBufferSize, + MaxActionsObserved: d.maxActionsObserved, + MaxActionsPerExecution: tweakables.MaxActionsPerExecution, + PendingStarts: make(map[string]struct{}), + StartRequested: d.startRequestsCopy(), + DroppedStarts: d.droppedStarts(), + } + d.ReadScheduler(func(s *scheduler.Scheduler, ctx chasm.Context) { + snap.ActionCount = s.Info.GetActionCount() + snap.Closed = s.Closed + snap.IsSentinel = s.IsSentinel() + snap.Paused = s.Schedule.GetState().GetPaused() + snap.BackfillerCount = len(s.Backfillers) + snap.OverlapSkipped = s.Info.GetOverlapSkipped() + snap.Migrating = s.GetWorkflowMigration() != nil + // Mirror Scheduler.isHeldOpen() from public state. + snap.IsHeldOpen = !snap.IsSentinel && (snap.Paused || snap.BackfillerCount > 0) + if s.IdleCloseTime != nil { + snap.IdleCloseTime = s.IdleCloseTime.AsTime() + } + // Sentinels have no generator/invoker sub-components. + if !snap.IsSentinel { + if g := s.Generator.Get(ctx); g != nil && g.LastProcessedTime != nil { + snap.GeneratorLPT = g.LastProcessedTime.AsTime() + } + if inv := s.Invoker.Get(ctx); inv != nil { + snap.UntransferableActions = len(inv.GetCancelWorkflows()) + len(inv.GetTerminateWorkflows()) + snap.BufferedStartsCount = len(inv.GetBufferedStarts()) + snap.addBufferedStarts(inv.GetBufferedStarts(), s.Schedule.GetPolicies().GetOverlapPolicy(), d.replacingSeen) + snap.ReplacingSeen = len(d.replacingSeen) + if inv.LastProcessedTime != nil { + snap.InvokerLPT = inv.LastProcessedTime.AsTime() + } + } + } + }) + return snap +} + +// addBufferedStarts classifies buffered starts into pending (deferred or not) +// and running. scheduleOverlap resolves starts with an unspecified policy. +// replacingSeen accumulates the request IDs of replacing starts across snapshots. +func (snap *Snapshot) addBufferedStarts(starts []*schedulespb.BufferedStart, scheduleOverlap enumspb.ScheduleOverlapPolicy, replacingSeen map[string]struct{}) { + for _, start := range starts { + policy := cmp.Or(start.GetOverlapPolicy(), scheduleOverlap) + replacing := policy == enumspb.SCHEDULE_OVERLAP_POLICY_CANCEL_OTHER || + policy == enumspb.SCHEDULE_OVERLAP_POLICY_TERMINATE_OTHER + if replacing { + replacingSeen[start.GetRequestId()] = struct{}{} + } + switch { + case start.GetRunId() == "": + if replacing && start.GetAttempt() <= 0 { + snap.ReplacingStarts++ + } + if start.GetAttempt() > 0 { + snap.UntransferableActions++ + } + snap.PendingStarts[start.GetRequestId()] = struct{}{} + if start.GetAttempt() == -1 { + snap.DeferredStarts++ + } + case start.GetCompleted() == nil: + snap.RunningWorkflows++ + default: + } + } +} + +// droppedStarts sums the ScheduleBufferedStartDropped counter. +func (d *Driver) droppedStarts() int64 { + var total int64 + for _, rec := range d.metricsCapture.SnapshotMetric(metrics.ScheduleBufferedStartDropped.Name()) { + if v, ok := rec.Value.(int64); ok { + total += v + } + } + return total +} diff --git a/chasm/lib/scheduler/schedulertest/task_validity_test.go b/chasm/lib/scheduler/schedulertest/task_validity_test.go index 5f8a4724544..bea44c7e8cb 100644 --- a/chasm/lib/scheduler/schedulertest/task_validity_test.go +++ b/chasm/lib/scheduler/schedulertest/task_validity_test.go @@ -22,7 +22,7 @@ func TestStalePhysicalTaskIsNotProgress(t *testing.T) { d := NewDriver(t, WithSchedule(schedule)) require.NoError(t, d.TriggerImmediately()) require.NotEmpty(t, d.runner.PendingSideEffectTasks()) - _, err := d.Engine.UpdateComponent(chasm.NewEngineContext(context.Background(), d.Engine), d.ref, + _, err := d.Engine.UpdateComponent(chasm.NewEngineContext(context.Background(), d.Engine), d.Ref, func(ctx chasm.MutableContext, c chasm.Component) error { c.(*scheduler.Scheduler).Invoker.Get(ctx).BufferedStarts = nil return nil @@ -44,7 +44,7 @@ func TestOldExecuteTaskCanProcessCurrentWork(t *testing.T) { tasks := d.runner.PendingSideEffectTasks() require.NotEmpty(t, tasks) old := tasks[0] - _, err := d.Engine.UpdateComponent(chasm.NewEngineContext(context.Background(), d.Engine), d.ref, + _, err := d.Engine.UpdateComponent(chasm.NewEngineContext(context.Background(), d.Engine), d.Ref, func(ctx chasm.MutableContext, c chasm.Component) error { inv := c.(*scheduler.Scheduler).Invoker.Get(ctx) current := common.CloneProto(inv.BufferedStarts[0]) @@ -73,7 +73,7 @@ func TestIdleDeadlineRequiresValidIdleTask(t *testing.T) { schedule.State.LimitedActions = true d := NewDriver(t, WithSchedule(schedule), WithoutStateInvariant()) require.Empty(t, d.StateViolations()) - _, err := d.Engine.UpdateComponent(chasm.NewEngineContext(context.Background(), d.Engine), d.ref, + _, err := d.Engine.UpdateComponent(chasm.NewEngineContext(context.Background(), d.Engine), d.Ref, func(ctx chasm.MutableContext, c chasm.Component) error { s := c.(*scheduler.Scheduler) s.LastEventTime = timestamppb.New(d.Now().Add(time.Hour)) @@ -94,7 +94,7 @@ func TestRetryTaskEqualityAndRearming(t *testing.T) { d := NewDriver(t, WithSchedule(schedule)) require.NoError(t, d.TriggerImmediately()) deadline := d.Now().Add(time.Minute) - _, err := d.Engine.UpdateComponent(chasm.NewEngineContext(context.Background(), d.Engine), d.ref, + _, err := d.Engine.UpdateComponent(chasm.NewEngineContext(context.Background(), d.Engine), d.Ref, func(ctx chasm.MutableContext, c chasm.Component) error { inv := c.(*scheduler.Scheduler).Invoker.Get(ctx) inv.BufferedStarts[0].Attempt = 2 From 6321164ff2aeba2073d69a96cbe9833ec71d4639 Mon Sep 17 00:00:00 2001 From: chaptersix <13949480+chaptersix@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:39:37 -0600 Subject: [PATCH 22/22] Describe the shared scheduler test scaffolding in the coverage document Co-Authored-By: Claude Fable 5.2 --- .../chasm-schedulertest-coverage-plan.md | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/docs/architecture/chasm-schedulertest-coverage-plan.md b/docs/architecture/chasm-schedulertest-coverage-plan.md index b221b5e4247..885cf43677c 100644 --- a/docs/architecture/chasm-schedulertest-coverage-plan.md +++ b/docs/architecture/chasm-schedulertest-coverage-plan.md @@ -124,10 +124,27 @@ Each fix has a default-run regression; none is gated. The component-agnostic stepping loop is `chasmtest.TaskRunner`: it finds the next pure or side-effect task, validates and dispatches it, advances the clock, -and exposes hooks for component-specific behaviour. `schedulertest` supplies the -scheduler's mocks, operations, `Snapshot` and invariants on top of it. The -engine, the runner and the driver take `chasmtest.TB`, which `*rapid.T` -satisfies, so a harness failure inside a property shrinks like any other. +and exposes hooks for component-specific behaviour. The engine, the runner and +the driver take `chasmtest.TB`, which `*rapid.T` satisfies, so a harness failure +inside a property shrinks like any other. + +`schedulertest` holds the one copy of the scheduler's test scaffolding, used by +both the unit tests in `chasm/lib/scheduler` and the lifecycle driver: + +- fixtures and library wiring (`DefaultSchedule`, `DefaultConfig`, + `NewTestLibrary`); +- `SchedulerEngine`, a `chasmtest.Engine` with typed `Read` and `Update`; +- `StateInvariant`, the stuck and idle-deadline checks, installed on every + engine through `chasmtest.WithInvariantCheck`. The idle-deadline check runs + armed idle tasks through the handler's `Validate`. + +The `Driver` embeds `SchedulerEngine` and adds the fake frontend and history +services, the runner hooks, the schedule operations, and `Snapshot` for the +invariants that need history or the start ledger. It keeps its own `no-stuck` +rule next to the shared one: the test engine never dispatches the framework +visibility task, which stays in the tree and hides a taskless schedule from the +shared count, while the shared rule has no exemption for paused or backfilling +schedules. `TestSchedulerProperties` runs as a Rapid state machine (`rt.Repeat`) over stepping, completion, pause, unpause, trigger, backfill, update, failed starts