diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 0a9fdd3d7c2..1015d1dbbfd 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -324,6 +324,33 @@ jobs: path: ${{ env.LEAK_OUTPUT_DIR }} if-no-files-found: ignore + property-test: + name: Property test + needs: [pre-build] + runs-on: ubuntu-24.04-arm + env: + GITHUB_TIMEOUT: 15 + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + ref: ${{ env.COMMIT }} + + - name: Prepare Go + uses: ./.github/actions/prepare-go + + - name: Run property tests + timeout-minutes: ${{ fromJSON(env.GITHUB_TIMEOUT) }} + run: make PROPERTY_TEST_TIMEOUT="$(("$GITHUB_TIMEOUT" - 1))m" property-test + + - name: Upload failing cases + if: failure() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: property-test-failfiles + path: "**/testdata/rapid/**/*.fail" + if-no-files-found: ignore + integration-test: name: Integration test needs: [pre-build] @@ -511,6 +538,7 @@ jobs: needs: - misc-checks - unit-test + - property-test - integration-test - functional-test runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index f6e1955edc7..dd6703b24bf 100644 --- a/.gitignore +++ b/.gitignore @@ -48,3 +48,7 @@ # Ignoring AI agent files .agents/ + +# rapid failfiles replay raw draw sequences and go stale on generator changes; +# pin regressions with hand-written tests. CI uploads them as artifacts on failure. +**/testdata/rapid/ diff --git a/Makefile b/Makefile index 6c881c70323..de665b51608 100644 --- a/Makefile +++ b/Makefile @@ -516,6 +516,29 @@ leak-test: -timeout $(LEAK_TIMEOUT) $(TEST_TAG_FLAG) \ ./tests/leakcheck/ -args -persistenceType=sql -persistenceDriver=sqlite +# Property tests get a dedicated run: more generated cases than the unit-test +# default, and no retries, so a failing seed is reported instead of retried away. +# Failing cases are minimized into testdata/rapid/ next to the test. +# Default: every package with a test file that imports pgregory.net/rapid (the +# -rapid.checks flag is only defined in test binaries that link rapid, so any +# other package would fail with "flag provided but not defined"). +PROPERTY_TEST_DIRS ?= $(patsubst %/,%,$(sort $(dir $(shell grep -l '"pgregory.net/rapid"' $(filter %_test.go,$(filter-out ./common/testx/% ./tests/mixedbrain/%,$(ALL_SRC))))))) +PROPERTY_TEST_CHECKS ?= 5000 +PROPERTY_TEST_TIMEOUT ?= 10m +property-test: + @printf $(COLOR) "Run property tests..." + CGO_ENABLED=$(TEST_CGO_ENABLED) go test $(PROPERTY_TEST_DIRS) -count=1 \ + -timeout $(PROPERTY_TEST_TIMEOUT) \ + $(if $(TEST_RACE_ENABLED),-race,) \ + $(TEST_TAG_FLAG) \ + -rapid.checks=$(PROPERTY_TEST_CHECKS) $(TEST_ARGS) + +# Mutation checks: each mutates one production file via a go test overlay and +# requires the named regression test to fail. See cmd/tools/mutationtest. +mutation-test: + @printf $(COLOR) "Run mutation tests..." + CGO_ENABLED=$(TEST_CGO_ENABLED) go run ./cmd/tools/mutationtest + verify-test-log: @test -s test.log || (echo "TEST FAILURE: test.log is missing or empty" && exit 1) @grep -q "^ok" test.log || (echo "TEST FAILURE: no passing test found in test.log" && exit 1) diff --git a/chasm/chasmtest/task_runner.go b/chasm/chasmtest/task_runner.go new file mode 100644 index 00000000000..3aea3c0cc7e --- /dev/null +++ b/chasm/chasmtest/task_runner.go @@ -0,0 +1,502 @@ +package chasmtest + +import ( + "cmp" + "context" + "slices" + "time" + + persistencespb "go.temporal.io/server/api/persistence/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/common/clock" + "go.temporal.io/server/service/history/tasks" + "google.golang.org/protobuf/proto" +) + +// TaskRunner steps one execution on an [Engine] through the tasks it schedules +// for itself, moving a controllable clock forward. It stands in for the queue +// processors that deliver pure and side-effect tasks in production, so a test +// can drive a component through its whole lifecycle deterministically and check +// invariants after every settled transition. +// +// A step fires every pure task due at the current reference time (through +// [Engine.FirePureTasks]) and dispatches every side-effect task whose +// visibility time has arrived, then advances the clock to the next scheduled +// task. A component that keeps re-arming a timer never quiesces, so callers +// bound [TaskRunner.RunToQuiescence] with a step count. +// +// Component-specific behaviour (completing external work before the clock +// passes some deadline, counting RPCs, tolerating particular task errors, +// intercepting the engine a task handler sees) is plugged in through +// [TaskRunnerHooks] and [WithAsyncEngine]. +// +// A TaskRunner is not safe for concurrent use. The exception is the +// Begin/Finish pair, which lets the test goroutine interleave with one task +// running on its own goroutine. +type TaskRunner struct { + t TB + engine *Engine + ref chasm.ComponentRef + clock *clock.EventTimeSource + + maxSideEffectLag time.Duration + hooks TaskRunnerHooks + wrapAsyncEngine func(chasm.Engine) chasm.Engine + + // dispatched holds the physical side-effect tasks already handed out, + // keyed by pointer identity: the engine's backend keeps every physical task + // it ever generated, and executing a task does not remove it. + dispatched map[*tasks.ChasmTask]struct{} + + // inFlight receives the result of the task started by BeginSideEffect. + inFlight chan error +} + +// TaskRunnerHooks are optional callbacks that let a component's test harness +// observe and extend the stepping loop. Any field may be nil. They run on the +// stepping goroutine, in the order the fields are listed for each kind of step. +// +// A side-effect dispatch runs BeforeSideEffect, executes the task, applies +// TolerateSideEffectError, runs AfterSideEffect, commits the transaction (which +// runs the engine's invariant checks), then runs AfterSideEffectCommit and +// AfterStep. A pure-task step fires the due tasks and commits, then runs +// AfterStep. BeforeAdvance runs ahead of every clock move. +type TaskRunnerHooks struct { + // BeforeAdvance runs before the clock moves to the given time and may + // itself move the clock to an earlier instant, for example to complete + // external work that would otherwise run past a deadline. It runs even + // when the clock does not end up moving. If it schedules work earlier than + // the given time, the runner stops there instead and calls it again. + BeforeAdvance func(to time.Time) + // BeforeSideEffect runs after a task has been selected and validated, just + // before it executes. + BeforeSideEffect func() + // AfterSideEffect runs once the task returned, successfully or with a + // tolerated error, before its transaction is committed. + AfterSideEffect func() + // AfterSideEffectCommit runs after a side-effect task's transaction is + // committed, before AfterStep. + AfterSideEffectCommit func() + // AfterStep runs after every committed pure or side-effect step. + AfterStep func() + // TolerateSideEffectError reports whether an error returned by a + // side-effect task is expected and should not fail the test. By default + // every error fails the test. + TolerateSideEffectError func(err error) bool +} + +// TaskRunnerOption configures a [TaskRunner]. +type TaskRunnerOption func(*TaskRunner) + +// WithMaxSideEffectLag bounds how far [TaskRunner.StepPure] may move the clock +// past the oldest undispatched side-effect task, modelling a side-effect queue +// that lags behind the timers. Zero, the default, means unbounded. +func WithMaxSideEffectLag(lag time.Duration) TaskRunnerOption { + return func(r *TaskRunner) { r.maxSideEffectLag = lag } +} + +// WithTaskRunnerHooks installs hooks. See [TaskRunnerHooks]. +func WithTaskRunnerHooks(hooks TaskRunnerHooks) TaskRunnerOption { + return func(r *TaskRunner) { r.hooks = hooks } +} + +// WithAsyncEngine wraps the engine that a task started with +// [TaskRunner.BeginSideEffect] sees, so a test can intercept the calls the +// handler makes back into the engine. Tasks dispatched through the Step methods +// always see the engine itself. +func WithAsyncEngine(wrap func(chasm.Engine) chasm.Engine) TaskRunnerOption { + return func(r *TaskRunner) { r.wrapAsyncEngine = wrap } +} + +// NewTaskRunner returns a runner for the execution identified by ref on e. The +// execution need not exist yet. ts must be the time source the engine was built +// with ([WithTimeSource]), since the runner moves it. +func NewTaskRunner( + t TB, + e *Engine, + ref chasm.ComponentRef, + ts *clock.EventTimeSource, + opts ...TaskRunnerOption, +) *TaskRunner { + r := &TaskRunner{ + t: t, + engine: e, + ref: ref, + clock: ts, + dispatched: make(map[*tasks.ChasmTask]struct{}), + } + for _, opt := range opts { + opt(r) + } + return r +} + +// Step advances the clock to the next scheduled task and fires all tasks due at +// that time: pure tasks first, then every due side-effect task. It returns false +// if nothing was pending, i.e. the execution is quiescent. +func (r *TaskRunner) Step() bool { + r.t.Helper() + + fireTime, ok := r.nextFireTime() + if !ok { + return false + } + r.advanceTo(fireTime, r.nextFireTime) + r.firePureTasks() + for r.StepSideEffect() { + } + return true +} + +// StepPure advances the clock to the next pending pure task and fires every +// pure task due then, leaving side-effect tasks undispatched. This models the +// side-effect queue lagging behind the timers. It returns false if no pure task +// is pending, or if firing it would move the clock more than the configured +// [WithMaxSideEffectLag] past the oldest undispatched side-effect task. +func (r *TaskRunner) StepPure() bool { + r.t.Helper() + + fireTime, ok := r.nextPureFireTime() + if !ok { + return false + } + if oldest, ok := r.oldestSideEffectTime(); ok && r.maxSideEffectLag > 0 && + fireTime.After(oldest.Add(r.maxSideEffectLag)) { + return false + } + r.advanceTo(fireTime, r.nextPureFireTime) + r.firePureTasks() + return true +} + +// StepSideEffect dispatches the oldest side-effect task that is due at the +// current time, without moving the clock. Tasks that are no longer valid are +// skipped. It returns false if none is due. +func (r *TaskRunner) StepSideEffect() bool { + r.t.Helper() + + next := r.nextSideEffect() + if next == nil { + return false + } + r.beforeSideEffect() + err := r.executeSideEffect(context.Background(), r.engine, next) + r.finishSideEffect(err) + return true +} + +// RunToQuiescence steps until no task is pending or maxSteps is reached. It +// returns the number of steps taken and whether the execution quiesced. +func (r *TaskRunner) RunToQuiescence(maxSteps int) (steps int, quiescent bool) { + r.t.Helper() + for steps = 0; steps < maxSteps; steps++ { + if !r.Step() { + return steps, true + } + } + return steps, false +} + +// HasPendingTask reports whether any task, pure or side-effect, is scheduled +// and still valid. +func (r *TaskRunner) HasPendingTask() bool { + r.t.Helper() + + if _, ok := r.nextPureFireTime(); ok { + return true + } + for _, task := range r.PendingSideEffectTasks() { + if inTree, valid := r.ValidateSideEffectTask(task); inTree && valid { + return true + } + } + return false +} + +// PendingPureTasks returns the pure tasks currently scheduled anywhere in the +// execution's tree, earliest first. The result is a copy. +func (r *TaskRunner) PendingPureTasks() []*persistencespb.ChasmComponentAttributes_Task { + r.t.Helper() + + var out []*persistencespb.ChasmComponentAttributes_Task + for _, node := range r.node().Snapshot(nil).Nodes { + for _, task := range node.GetMetadata().GetComponentAttributes().GetPureTasks() { + out = append(out, proto.Clone(task).(*persistencespb.ChasmComponentAttributes_Task)) + } + } + slices.SortStableFunc(out, func(a, b *persistencespb.ChasmComponentAttributes_Task) int { + return a.GetScheduledTime().AsTime().Compare(b.GetScheduledTime().AsTime()) + }) + return out +} + +// BeginSideEffect selects the next due side-effect task and runs it on its own +// goroutine, through the engine wrapped by [WithAsyncEngine], so the test can +// interleave with its calls back into the engine. It returns false if no task +// is due. The result arrives on [TaskRunner.SideEffectDone]; pass it to +// [TaskRunner.FinishSideEffect]. Only one task may be in flight, and the test +// may touch the engine meanwhile only while the handler is blocked in its +// wrapper. +func (r *TaskRunner) BeginSideEffect() bool { + r.t.Helper() + + if r.inFlight != nil { + r.t.Fatalf("a side-effect task is already in flight") + } + next := r.nextSideEffect() + if next == nil { + return false + } + r.beforeSideEffect() + done := make(chan error, 1) + r.inFlight = done + ctx, cancel := context.WithCancel(r.t.Context()) + r.t.Cleanup(func() { + cancel() + if r.inFlight == done { + <-done + r.inFlight = nil + } + }) + var engine chasm.Engine = r.engine + if r.wrapAsyncEngine != nil { + engine = r.wrapAsyncEngine(engine) + } + go func() { done <- r.executeSideEffect(ctx, engine, next) }() + return true +} + +// SideEffectDone returns the channel on which the task started by +// [TaskRunner.BeginSideEffect] reports its result, or nil if none is in flight. +func (r *TaskRunner) SideEffectDone() <-chan error { + return r.inFlight +} + +// FinishSideEffect completes the in-flight task after its result err was +// received from [TaskRunner.SideEffectDone]: it checks the error, commits the +// transaction and runs the after-step hooks. +func (r *TaskRunner) FinishSideEffect(err error) { + r.t.Helper() + r.inFlight = nil + r.finishSideEffect(err) +} + +// AbandonSideEffect forgets the in-flight task after its result was received +// from [TaskRunner.SideEffectDone] but will not be finished, typically because +// the test is about to fail. +func (r *TaskRunner) AbandonSideEffect() { + r.inFlight = nil +} + +func (r *TaskRunner) node() *chasm.Node { + r.t.Helper() + exec, err := r.engine.executionForRef(r.ref) + if err != nil { + r.t.Fatalf("execution lookup failed: %v", err) + } + return exec.node +} + +// nextPureFireTime returns the earliest scheduled pure task time in the tree. +// +// The tree's own pure-task metadata is the source rather than the engine +// backend's record of the last physical-task deletion: when a transaction +// closes, the node derives that recorded time from exactly this minimum over the +// first pure task of each component, so the two agree whenever the tree is +// clean, which it is between steps. +func (r *TaskRunner) nextPureFireTime() (time.Time, bool) { + r.t.Helper() + var earliest time.Time + have := false + for _, node := range r.node().Snapshot(nil).Nodes { + pure := node.GetMetadata().GetComponentAttributes().GetPureTasks() + if len(pure) == 0 { + continue + } + if at := pure[0].GetScheduledTime().AsTime(); !have || at.Before(earliest) { + earliest, have = at, true + } + } + return earliest, have +} + +// oldestSideEffectTime returns the earliest visibility time among undispatched +// side-effect tasks, if any. +func (r *TaskRunner) oldestSideEffectTime() (time.Time, bool) { + var oldest time.Time + have := false + for _, task := range r.PendingSideEffectTasks() { + if vt := task.GetVisibilityTime(); !have || vt.Before(oldest) { + oldest, have = vt, true + } + } + return oldest, have +} + +// nextFireTime returns the earliest time at which any pending task is due. +func (r *TaskRunner) nextFireTime() (time.Time, bool) { + pure, havePure := r.nextPureFireTime() + side, haveSide := r.oldestSideEffectTime() + switch { + case havePure && haveSide: + if side.Before(pure) { + return side, true + } + return pure, true + case havePure: + return pure, true + default: + return side, haveSide + } +} + +// PendingSideEffectTasks returns the physical side-effect tasks this runner has +// not yet handed out, in the order the production queue would deliver them. A +// task may be pending yet no longer valid; see [TaskRunner.ValidateSideEffectTask]. +func (r *TaskRunner) PendingSideEffectTasks() []*tasks.ChasmTask { + exec, err := r.engine.executionForRef(r.ref) + if err != nil { + r.t.Fatalf("execution lookup failed: %v", err) + } + var out []*tasks.ChasmTask + for category, categoryTasks := range exec.backend.TasksByCategory { + // Visibility tasks are framework-managed (indexing search attributes) and + // processed by a separate queue, not the CHASM side-effect executor; their + // handler panics if invoked directly. Skip them. + if category.ID() == tasks.CategoryVisibility.ID() { + continue + } + for _, t := range categoryTasks { + ct, ok := t.(*tasks.ChasmTask) + if !ok { + continue + } + if _, done := r.dispatched[ct]; done { + continue + } + out = append(out, ct) + } + } + slices.SortStableFunc(out, func(a, b *tasks.ChasmTask) int { + if c := a.GetVisibilityTime().Compare(b.GetVisibilityTime()); c != 0 { + return c + } + if c := cmp.Compare(a.Info.GetTaskVersionedTransition().GetTransitionCount(), b.Info.GetTaskVersionedTransition().GetTransitionCount()); c != 0 { + return c + } + if c := cmp.Compare(a.Info.GetTaskVersionedTransitionOffset(), b.Info.GetTaskVersionedTransitionOffset()); c != 0 { + return c + } + if c := cmp.Compare(a.Info.GetTypeId(), b.Info.GetTypeId()); c != 0 { + return c + } + return slices.Compare(a.Info.GetPath(), b.Info.GetPath()) + }) + return out +} + +// ValidateSideEffectTask reports whether the physical task still corresponds to +// a logical task in the tree (inTree), and whether that logical task's handler +// still considers it valid. It does not mark the task as dispatched. +func (r *TaskRunner) ValidateSideEffectTask(task *tasks.ChasmTask) (inTree, valid bool) { + r.t.Helper() + inTree, valid, err := r.node().ValidateSideEffectTask(context.Background(), task) + if err != nil { + r.t.Fatalf("ValidateSideEffectTask: %v", err) + } + return inTree, valid +} + +// nextSideEffect returns the oldest due, valid side-effect task and marks it +// and every invalid task before it as dispatched. +func (r *TaskRunner) nextSideEffect() *tasks.ChasmTask { + r.t.Helper() + for _, task := range r.PendingSideEffectTasks() { + if task.GetVisibilityTime().After(r.clock.Now()) { + continue + } + r.dispatched[task] = struct{}{} + if inTree, valid := r.ValidateSideEffectTask(task); inTree && valid { + return task + } + } + return nil +} + +func (r *TaskRunner) executeSideEffect(ctx context.Context, engine chasm.Engine, task *tasks.ChasmTask) error { + return r.node().ExecuteSideEffectTask( + chasm.NewEngineContext(ctx, engine), r.ref.ExecutionKey, task, + func(chasm.NodeBackend, chasm.Context, chasm.Component) error { return nil }, + ) +} + +func (r *TaskRunner) beforeSideEffect() { + if r.hooks.BeforeSideEffect != nil { + r.hooks.BeforeSideEffect() + } +} + +func (r *TaskRunner) finishSideEffect(err error) { + r.t.Helper() + if err != nil && (r.hooks.TolerateSideEffectError == nil || !r.hooks.TolerateSideEffectError(err)) { + r.t.Fatalf("ExecuteSideEffectTask failed: %v", err) + } + if r.hooks.AfterSideEffect != nil { + r.hooks.AfterSideEffect() + } + r.commit() + if r.hooks.AfterSideEffectCommit != nil { + r.hooks.AfterSideEffectCommit() + } + r.afterStep() +} + +// firePureTasks fires every pure task due at the current time and commits. +func (r *TaskRunner) firePureTasks() { + r.t.Helper() + if _, err := r.engine.FirePureTasks(r.ref, r.clock.Now()); err != nil { + r.t.Fatalf("FirePureTasks failed: %v", err) + } + r.afterStep() +} + +func (r *TaskRunner) commit() { + r.t.Helper() + exec, err := r.engine.executionForRef(r.ref) + if err == nil { + err = r.engine.closeTransaction(context.Background(), exec) + } + if err != nil { + r.t.Fatalf("closing transaction: %v", err) + } +} + +func (r *TaskRunner) afterStep() { + if r.hooks.AfterStep != nil { + r.hooks.AfterStep() + } +} + +// advanceTo moves the clock to t, never backward, after giving BeforeAdvance a +// chance to act on the way. BeforeAdvance may schedule work that falls before +// t, such as the retry of a start it dispatched; next reports the earliest +// pending fire time, and the clock stops there instead of jumping over it. +func (r *TaskRunner) advanceTo(t time.Time, next func() (time.Time, bool)) { + for r.hooks.BeforeAdvance != nil { + r.hooks.BeforeAdvance(t) + earliest, ok := next() + if !ok { + break + } + if now := r.clock.Now(); earliest.Before(now) { + earliest = now + } + if !earliest.Before(t) { + break + } + t = earliest + } + if t.After(r.clock.Now()) { + r.clock.Update(t) + } +} diff --git a/chasm/chasmtest/task_runner_test.go b/chasm/chasmtest/task_runner_test.go new file mode 100644 index 00000000000..04716607616 --- /dev/null +++ b/chasm/chasmtest/task_runner_test.go @@ -0,0 +1,304 @@ +package chasmtest_test + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.temporal.io/api/serviceerror" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/chasmtest" + "go.temporal.io/server/chasm/lib/tests" + "go.temporal.io/server/chasm/lib/tests/gen/testspb/v1" + "go.temporal.io/server/common/clock" + "go.temporal.io/server/common/log" + "google.golang.org/protobuf/types/known/timestamppb" +) + +var runnerStart = time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) + +type runnerFixture struct { + engine *chasmtest.Engine + ref chasm.ComponentRef + ts *clock.EventTimeSource +} + +// newRunnerFixture starts a payload store holding payload "ttl", whose pure +// task removes it after ttl. +func newRunnerFixture(t *testing.T, ttl time.Duration) *runnerFixture { + t.Helper() + registry := chasm.NewRegistry(log.NewNoopLogger()) + require.NoError(t, registry.Register(&chasm.CoreLibrary{})) + require.NoError(t, registry.Register(tests.Library)) + ts := clock.NewEventTimeSource() + ts.Update(runnerStart) + engine := chasmtest.NewEngine(t, registry, chasmtest.WithTimeSource(ts)) + + key := chasm.ExecutionKey{NamespaceID: "test-ns", BusinessID: "store"} + _, err := chasm.StartExecution(engineContext(engine), key, + func(mc chasm.MutableContext, _ any) (*tests.PayloadStore, error) { + store, err := tests.NewPayloadStore(mc) + if err != nil { + return nil, err + } + return store, addPayload(store, mc, "ttl", ttl) + }, nil) + require.NoError(t, err) + return &runnerFixture{engine: engine, ref: chasm.NewComponentRef[*tests.PayloadStore](key), ts: ts} +} + +// addSideEffect schedules a side-effect task for key at the given offset from +// the start. When present is false the key has no payload, so the task is valid +// but its execution fails with NotFound. +func (f *runnerFixture) addSideEffect(t *testing.T, key string, after time.Duration, present bool) { + t.Helper() + _, err := f.engine.UpdateComponent(engineContext(f.engine), f.ref, + func(mc chasm.MutableContext, c chasm.Component) error { + store := c.(*tests.PayloadStore) + if present { + if err := addPayload(store, mc, key, 0); err != nil { + return err + } + } + at := runnerStart.Add(after) + if store.State.ExpirationTimes == nil { + store.State.ExpirationTimes = make(map[string]*timestamppb.Timestamp) + } + store.State.ExpirationTimes[key] = timestamppb.New(at) + mc.AddTask(store, chasm.TaskAttributes{ScheduledTime: at}, &testspb.TestPayloadTTLSideEffectTask{PayloadKey: key}) + return nil + }) + require.NoError(t, err) +} + +func (f *runnerFixture) hasPayload(t *testing.T, key string) bool { + t.Helper() + var ok bool + require.NoError(t, f.engine.ReadComponent(engineContext(f.engine), f.ref, + func(_ chasm.Context, c chasm.Component) error { + _, ok = c.(*tests.PayloadStore).Payloads[key] + return nil + })) + return ok +} + +func TestTaskRunnerStepAdvancesClockAndFiresPureTasks(t *testing.T) { + f := newRunnerFixture(t, time.Hour) + r := chasmtest.NewTaskRunner(t, f.engine, f.ref, f.ts) + + require.True(t, r.HasPendingTask()) + require.True(t, f.hasPayload(t, "ttl")) + + require.True(t, r.Step()) + require.Equal(t, runnerStart.Add(time.Hour), f.ts.Now()) + require.False(t, f.hasPayload(t, "ttl")) + + require.False(t, r.HasPendingTask()) + require.False(t, r.Step()) + require.Equal(t, runnerStart.Add(time.Hour), f.ts.Now(), "a quiescent step leaves the clock alone") +} + +func TestTaskRunnerRunToQuiescence(t *testing.T) { + f := newRunnerFixture(t, time.Hour) + f.addSideEffect(t, "effect", 2*time.Hour, true) + r := chasmtest.NewTaskRunner(t, f.engine, f.ref, f.ts) + + steps, quiescent := r.RunToQuiescence(10) + require.True(t, quiescent) + require.Equal(t, 2, steps) + require.False(t, f.hasPayload(t, "effect")) + + f = newRunnerFixture(t, time.Hour) + f.addSideEffect(t, "effect", 2*time.Hour, true) + r = chasmtest.NewTaskRunner(t, f.engine, f.ref, f.ts) + steps, quiescent = r.RunToQuiescence(1) + require.False(t, quiescent) + require.Equal(t, 1, steps) +} + +func TestTaskRunnerPendingPureTasksAreEarliestFirst(t *testing.T) { + f := newRunnerFixture(t, time.Hour) + _, err := f.engine.UpdateComponent(engineContext(f.engine), f.ref, + func(mc chasm.MutableContext, c chasm.Component) error { + return addPayload(c.(*tests.PayloadStore), mc, "sooner", time.Minute) + }) + require.NoError(t, err) + r := chasmtest.NewTaskRunner(t, f.engine, f.ref, f.ts) + + var scheduled []time.Time + for _, task := range r.PendingPureTasks() { + scheduled = append(scheduled, task.GetScheduledTime().AsTime()) + } + require.Equal(t, []time.Time{runnerStart.Add(time.Minute), runnerStart.Add(time.Hour)}, scheduled) +} + +func TestTaskRunnerStepPureLeavesSideEffectsUndispatched(t *testing.T) { + for _, tc := range []struct { + name string + lag time.Duration + wantStepPure bool + }{ + {name: "unbounded lag", lag: 0, wantStepPure: true}, + {name: "lag shorter than the gap", lag: 5 * time.Minute, wantStepPure: false}, + {name: "lag longer than the gap", lag: 2 * time.Hour, wantStepPure: true}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newRunnerFixture(t, time.Hour) + f.addSideEffect(t, "effect", 10*time.Minute, true) + r := chasmtest.NewTaskRunner(t, f.engine, f.ref, f.ts, chasmtest.WithMaxSideEffectLag(tc.lag)) + + require.Equal(t, tc.wantStepPure, r.StepPure()) + if tc.wantStepPure { + require.Equal(t, runnerStart.Add(time.Hour), f.ts.Now()) + require.False(t, f.hasPayload(t, "ttl")) + } else { + require.Equal(t, runnerStart, f.ts.Now()) + } + require.True(t, f.hasPayload(t, "effect"), "StepPure never dispatches side-effect tasks") + }) + } +} + +func TestTaskRunnerStepSideEffectDispatchesOnlyDueTasks(t *testing.T) { + f := newRunnerFixture(t, 0) + f.addSideEffect(t, "effect", 10*time.Minute, true) + r := chasmtest.NewTaskRunner(t, f.engine, f.ref, f.ts) + + require.False(t, r.StepSideEffect(), "not due yet") + require.True(t, f.hasPayload(t, "effect")) + + f.ts.Update(runnerStart.Add(10 * time.Minute)) + require.True(t, r.StepSideEffect()) + require.False(t, f.hasPayload(t, "effect")) + require.False(t, r.StepSideEffect(), "a dispatched task is not dispatched again") +} + +func TestTaskRunnerHookOrder(t *testing.T) { + f := newRunnerFixture(t, time.Hour) + f.addSideEffect(t, "effect", time.Hour, true) + + var events []string + record := func(name string) func() { return func() { events = append(events, name) } } + var advanceTo, clockAtAdvance time.Time + r := chasmtest.NewTaskRunner(t, f.engine, f.ref, f.ts, chasmtest.WithTaskRunnerHooks(chasmtest.TaskRunnerHooks{ + BeforeAdvance: func(to time.Time) { + advanceTo, clockAtAdvance = to, f.ts.Now() + events = append(events, "beforeAdvance") + }, + BeforeSideEffect: record("beforeSideEffect"), + AfterSideEffect: record("afterSideEffect"), + AfterSideEffectCommit: record("afterSideEffectCommit"), + AfterStep: record("afterStep"), + })) + + require.True(t, r.Step()) + require.Equal(t, runnerStart.Add(time.Hour), advanceTo) + require.Equal(t, runnerStart, clockAtAdvance, "BeforeAdvance runs before the clock moves") + require.Equal(t, []string{ + "beforeAdvance", + "afterStep", // pure tasks + "beforeSideEffect", "afterSideEffect", "afterSideEffectCommit", "afterStep", + }, events) +} + +func TestTaskRunnerBeforeAdvanceMayMoveClock(t *testing.T) { + f := newRunnerFixture(t, time.Hour) + var removedBeforeMove bool + r := chasmtest.NewTaskRunner(t, f.engine, f.ref, f.ts, chasmtest.WithTaskRunnerHooks(chasmtest.TaskRunnerHooks{ + BeforeAdvance: func(to time.Time) { + f.ts.Update(to.Add(-30 * time.Minute)) + removedBeforeMove = !f.hasPayload(t, "ttl") + }, + })) + require.True(t, r.Step()) + require.False(t, removedBeforeMove) + require.Equal(t, runnerStart.Add(time.Hour), f.ts.Now(), "the runner still lands on the task's time") +} + +func TestTaskRunnerBeforeAdvanceSchedulesEarlierWork(t *testing.T) { + f := newRunnerFixture(t, time.Hour) + var advances []time.Time + var dispatchedAt []time.Time + r := chasmtest.NewTaskRunner(t, f.engine, f.ref, f.ts, chasmtest.WithTaskRunnerHooks(chasmtest.TaskRunnerHooks{ + BeforeAdvance: func(to time.Time) { + advances = append(advances, to) + if len(advances) == 1 { + f.addSideEffect(t, "retry", 10*time.Minute, true) + } + }, + AfterSideEffect: func() { dispatchedAt = append(dispatchedAt, f.ts.Now()) }, + })) + + require.True(t, r.Step()) + require.Equal(t, []time.Time{runnerStart.Add(time.Hour), runnerStart.Add(10 * time.Minute)}, advances) + require.Equal(t, []time.Time{runnerStart.Add(10 * time.Minute)}, dispatchedAt, + "work the hook scheduled before the target time is not jumped over") + require.Equal(t, runnerStart.Add(10*time.Minute), f.ts.Now()) +} + +func TestTaskRunnerToleratedSideEffectError(t *testing.T) { + f := newRunnerFixture(t, 0) + f.addSideEffect(t, "ghost", time.Minute, false) + f.ts.Update(runnerStart.Add(time.Minute)) + + var tolerated error + var afterSteps int + r := chasmtest.NewTaskRunner(t, f.engine, f.ref, f.ts, chasmtest.WithTaskRunnerHooks(chasmtest.TaskRunnerHooks{ + TolerateSideEffectError: func(err error) bool { + tolerated = err + _, ok := errors.AsType[*serviceerror.NotFound](err) + return ok + }, + AfterStep: func() { afterSteps++ }, + })) + + require.True(t, r.StepSideEffect()) + require.Error(t, tolerated) + require.Equal(t, 1, afterSteps, "a tolerated error still settles the step") +} + +func TestTaskRunnerBeginAndFinishSideEffect(t *testing.T) { + f := newRunnerFixture(t, 0) + f.addSideEffect(t, "effect", time.Minute, true) + + var wrapped, afterSteps int + r := chasmtest.NewTaskRunner(t, f.engine, f.ref, f.ts, + chasmtest.WithAsyncEngine(func(e chasm.Engine) chasm.Engine { + return &countingEngine{Engine: e, updates: &wrapped} + }), + chasmtest.WithTaskRunnerHooks(chasmtest.TaskRunnerHooks{AfterStep: func() { afterSteps++ }}), + ) + + require.Nil(t, r.SideEffectDone()) + require.False(t, r.BeginSideEffect(), "nothing is due yet") + + f.ts.Update(runnerStart.Add(time.Minute)) + require.True(t, r.BeginSideEffect()) + err := <-r.SideEffectDone() + require.NoError(t, err) + require.Equal(t, 0, afterSteps, "the step settles only when finished") + r.FinishSideEffect(err) + + require.Nil(t, r.SideEffectDone()) + require.Equal(t, 1, afterSteps) + require.Equal(t, 1, wrapped, "the handler saw the wrapped engine") + require.False(t, f.hasPayload(t, "effect")) + require.False(t, r.BeginSideEffect()) +} + +type countingEngine struct { + chasm.Engine + updates *int +} + +func (e *countingEngine) UpdateComponent( + ctx context.Context, + ref chasm.ComponentRef, + update func(chasm.MutableContext, chasm.Component) error, + opts ...chasm.TransitionOption, +) ([]byte, error) { + *e.updates++ + return e.Engine.UpdateComponent(ctx, ref, update, opts...) +} diff --git a/chasm/chasmtest/test_engine.go b/chasm/chasmtest/test_engine.go index 4724d374c0f..fd836f85de5 100644 --- a/chasm/chasmtest/test_engine.go +++ b/chasm/chasmtest/test_engine.go @@ -6,7 +6,6 @@ import ( "fmt" "maps" "sync" - "testing" "time" enumspb "go.temporal.io/api/enums/v1" @@ -25,19 +24,30 @@ import ( ) type ( + // TB is the subset of [testing.TB] the engine, gomock controllers and + // testlogger need. Both *testing.T and *rapid.T satisfy it, so an engine can + // be built inside a rapid property and have failures flow through rapid's + // shrinking. It is declared here, rather than imported from rapid, so this + // package does not depend on rapid. + TB interface { + testlogger.CleanupCapableT + Errorf(format string, args ...any) + Context() context.Context + } + EngineOption func(*Engine) // InvariantCheck validates an execution after a successful transition. The // node is clean, so checks may inspect its snapshot without affecting the // state being validated. - InvariantCheck func(*testing.T, *chasm.Node, chasm.RootComponent) + InvariantCheck func(TB, *chasm.Node, chasm.RootComponent) // Engine is a lightweight in memory CHASM engine for unit tests. It implements // [chasm.Engine] and supports the full set of conflict and reuse policies, as // well as blocking [PollComponent] with [NotifyExecution], matching the behavior // of the production engine as closely as possible without persistence or shard logic. Engine struct { - t *testing.T + t TB registry *chasm.Registry logger log.Logger metrics metrics.Handler @@ -107,7 +117,7 @@ var defaultTransitionOptions = chasm.TransitionOptions{ var _ chasm.Engine = (*Engine)(nil) func NewEngine( - t *testing.T, + t TB, registry *chasm.Registry, opts ...EngineOption, ) *Engine { diff --git a/chasm/chasmtest/test_engine_test.go b/chasm/chasmtest/test_engine_test.go index 48dc3c54708..390595d2ef7 100644 --- a/chasm/chasmtest/test_engine_test.go +++ b/chasm/chasmtest/test_engine_test.go @@ -50,7 +50,7 @@ func TestTasksArePhysicallyGenerated(t *testing.T) { func TestInvariantCheckRunsAfterSuccessfulTransitions(t *testing.T) { var calls []string check := func(name string) chasmtest.InvariantCheck { - return func(checkT *testing.T, node *chasm.Node, root chasm.RootComponent) { + return func(checkT chasmtest.TB, node *chasm.Node, root chasm.RootComponent) { checkT.Helper() require.Same(checkT, t, checkT) require.IsType(checkT, &tests.PayloadStore{}, root) @@ -122,7 +122,7 @@ func TestInvariantCheckRunsAfterSuccessfulTransitions(t *testing.T) { // sub-components its update adds must become nodes in the tree, not only exist in memory. func TestUpdateWithStartSyncsSubComponentsAddedByUpdate(t *testing.T) { var nodePaths []string - check := func(_ *testing.T, node *chasm.Node, _ chasm.RootComponent) { + check := func(_ chasmtest.TB, node *chasm.Node, _ chasm.RootComponent) { nodePaths = slices.Collect(maps.Keys(node.Snapshot(nil).Nodes)) } e, _ := startStoreWithEngineOptions(t, 0, []chasmtest.EngineOption{chasmtest.WithInvariantCheck(check)}) @@ -368,7 +368,7 @@ func TestRecordsCallerPrincipal(t *testing.T) { worker := &commonpb.Principal{Type: "jwt", Name: "worker"} var rootAttributes *persistencespb.ChasmComponentAttributes - captureRoot := func(_ *testing.T, node *chasm.Node, _ chasm.RootComponent) { + captureRoot := func(_ chasmtest.TB, node *chasm.Node, _ chasm.RootComponent) { rootAttributes = node.Snapshot(nil).Nodes[""].GetMetadata().GetComponentAttributes() } diff --git a/chasm/lib/scheduler/backfill_capacity_progress_test.go b/chasm/lib/scheduler/backfill_capacity_progress_test.go new file mode 100644 index 00000000000..4b986005a1f --- /dev/null +++ b/chasm/lib/scheduler/backfill_capacity_progress_test.go @@ -0,0 +1,69 @@ +package scheduler_test + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + enumspb "go.temporal.io/api/enums/v1" + schedulepb "go.temporal.io/api/schedule/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/scheduler" + "go.temporal.io/server/common/clock" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestBackfillCapacityNativeControl(t *testing.T) { + testBackfillCapacityProgress(t, 450) +} + +func TestBackfillCapacityProgress(t *testing.T) { + for _, n := range []int{451, 1000} { + t.Run(fmt.Sprint(n), func(t *testing.T) { testBackfillCapacityProgress(t, n) }) + } +} + +func testBackfillCapacityProgress(t *testing.T, count int) { + t.Helper() + now := time.Date(2026, 9, 4, 12, 0, 0, 0, time.UTC) + ts := clock.NewEventTimeSource().Update(now) + spec := defaultSchedule() + spec.State.Paused = true + e := newSchedulerTestEngine(t, spec, withEngineTimeSource(ts)) + require.NoError(t, e.updateScheduler(func(s *scheduler.Scheduler, ctx chasm.MutableContext) error { + for range count { + s.NewRangeBackfiller(ctx, &schedulepb.BackfillRequest{ + StartTime: timestamppb.New(now), EndTime: timestamppb.New(now), + OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL, + }) + } + return nil + })) + seen := make(map[string]string) + remaining := count + for round := 0; round < 50 && remaining > 0; round++ { + buffered := 0 + require.NoError(t, e.updateScheduler(func(s *scheduler.Scheduler, ctx chasm.MutableContext) error { + i := s.Invoker.Get(ctx) + buffered = len(i.BufferedStarts) + require.LessOrEqual(t, buffered, 460, "shared half-buffer minus generator reserve plus retained-history allowance") + for _, start := range i.BufferedStarts { + require.NotContains(t, seen, start.RequestId) + seen[start.RequestId] = start.WorkflowId + } + i.BufferedStarts = nil + remaining = len(s.Backfillers) + return nil + })) + if remaining == 0 { + break + } + require.Positive(t, buffered, "count=%d: empty buffer and %d ranges must make progress", count, remaining) + ts.Update(ts.Now().Add(time.Hour)) + _, err := e.engine.FirePureTasks(e.rootRef, ts.Now()) + require.NoError(t, err) + } + require.Zero(t, remaining) + require.Len(t, seen, count) +} diff --git a/chasm/lib/scheduler/backfiller.go b/chasm/lib/scheduler/backfiller.go index ba26dcb4b82..72b28b7e977 100644 --- a/chasm/lib/scheduler/backfiller.go +++ b/chasm/lib/scheduler/backfiller.go @@ -37,7 +37,7 @@ func addBackfiller( ) *Backfiller { id := schedulerinternal.GenerateBackfillerID() // LastProcessedTime is intentionally left unset here. For range backfills it - // is the "progress recorded" signal and must stay zero until a batch is + // is the "progress recorded" signal and must stay nil until a batch is // actually processed (see processBackfill). Trigger backfills, which use it as // their fire time, set it explicitly in NewImmediateBackfiller. backfiller := newBackfillerWithState(ctx, &schedulerpb.BackfillerState{ @@ -55,12 +55,27 @@ func addBackfiller( } func newBackfillerWithState(ctx chasm.MutableContext, state *schedulerpb.BackfillerState) *Backfiller { - backfiller := &Backfiller{ + backfiller := newBackfillerWithStateWithoutTask(ctx, state) + backfiller.scheduleInitialTask(ctx) + return backfiller +} + +// newBackfillerWithStateWithoutTask returns a Backfiller that has no task +// scheduled. Call scheduleInitialTask to start it. +func newBackfillerWithStateWithoutTask(ctx chasm.MutableContext, state *schedulerpb.BackfillerState) *Backfiller { + return &Backfiller{ BackfillerState: state, EventLog: chasm.NewComponentField(ctx, NewEventLog(ctx)), } - backfiller.scheduleTask(ctx, chasm.TaskScheduledTimeImmediate) - return backfiller +} + +// scheduleInitialTask schedules the first immediate task, unless a task was +// already scheduled. +func (b *Backfiller) scheduleInitialTask(ctx chasm.MutableContext) { + if b.GetTaskStamp() != 0 { + return + } + b.scheduleTask(ctx, chasm.TaskScheduledTimeImmediate) } // scheduleTask advances the stamp for the initial task or a successor. diff --git a/chasm/lib/scheduler/backfiller_capacity_internal_test.go b/chasm/lib/scheduler/backfiller_capacity_internal_test.go index c697aa0019b..29e852ad6b8 100644 --- a/chasm/lib/scheduler/backfiller_capacity_internal_test.go +++ b/chasm/lib/scheduler/backfiller_capacity_internal_test.go @@ -28,7 +28,7 @@ import ( // // pending = max(0, bufferedCount - retainedActionCount) // available = max(0, maxBufferSize/2 - pending - generatorReserve) -// result = available / max(1, backfillerCount) +// result = min(available, max(1, available / max(1, backfillerCount))) func TestBackfillerBufferCapacity(t *testing.T) { cases := []struct { name string @@ -77,7 +77,7 @@ func TestBackfillerBufferCapacity(t *testing.T) { {"single backfiller gets the whole remainder", 0, 0, 1000, 50, 1, 450}, {"ten backfillers each get an even share (regression)", 0, 0, 1000, 50, 10, 45}, {"backfillers exactly dividing remainder get one each", 0, 0, 1000, 50, 450, 1}, - {"more backfillers than remainder truncate to zero", 0, 0, 1000, 50, 451, 0}, + {"more backfillers than remainder still progress", 0, 0, 1000, 50, 451, 1}, {"zero backfillers are clamped to one", 0, 0, 1000, 50, 0, 450}, {"negative backfillers are clamped to one", 0, 0, 1000, 50, -3, 450}, } diff --git a/chasm/lib/scheduler/backfiller_tasks.go b/chasm/lib/scheduler/backfiller_tasks.go index 082215cbfcc..c68d5392208 100644 --- a/chasm/lib/scheduler/backfiller_tasks.go +++ b/chasm/lib/scheduler/backfiller_tasks.go @@ -176,11 +176,13 @@ func (b *BackfillerTaskHandler) processBackfill( endTime := request.GetEndTime().AsTime() // Resume from the high watermark only once genuine progress has been recorded. - // The watermark is left unset until a batch is actually processed (see Execute), - // so a fresh or capacity-stalled backfiller starts from the range start. + // The watermark stays nil until a batch is actually processed (see Execute) or + // a V1 cursor is imported, so a fresh or capacity-stalled backfiller starts + // from the range start. Presence, not value, matters: an explicit Unix epoch + // cursor is real progress. var startTime time.Time lastProcessed := backfiller.GetLastProcessedTime() - if hasRecordedProgress(lastProcessed) { + if lastProcessed != nil { startTime = lastProcessed.AsTime() } else { // On the first attempt, start slightly behind to make the range inclusive. @@ -213,13 +215,6 @@ func (b *BackfillerTaskHandler) processBackfill( return } -// hasRecordedProgress reports whether a backfiller's high watermark reflects a -// batch that was actually processed. An unset (nil or zero) watermark means no -// progress yet - a fresh backfiller. -func hasRecordedProgress(lastProcessed *timestamppb.Timestamp) bool { - return lastProcessed != nil && (lastProcessed.GetSeconds() != 0 || lastProcessed.GetNanos() != 0) -} - // backoffDelay returns the amount of delay that should be added when retrying. func (b *BackfillerTaskHandler) backoffDelay(backfiller *Backfiller) time.Duration { // Increment GetAttempt here early, to avoid needing to increment @@ -293,5 +288,6 @@ func backfillerBufferCapacity(bufferedCount, retainedActionCount, maxBufferSize, backfillerCount = max(1, backfillerCount) pending := max(0, bufferedCount-retainedActionCount) available := max(0, (maxBufferSize/2)-pending-generatorReserve) - return available / backfillerCount + // Pure tasks serialize on the tree and recompute the shared budget after each enqueue. + return min(available, max(1, available/backfillerCount)) } diff --git a/chasm/lib/scheduler/completion.go b/chasm/lib/scheduler/completion.go index 30208d05123..2ba8cc6ec03 100644 --- a/chasm/lib/scheduler/completion.go +++ b/chasm/lib/scheduler/completion.go @@ -20,6 +20,7 @@ import ( const ( callbackIgnoredUnrecognizedRequest metrics.ReasonString = "unrecognized_request_id" callbackIgnoredAlreadyCompleted metrics.ReasonString = "already_completed" + callbackIgnoredStartNotRecorded metrics.ReasonString = "start_not_recorded" ) var _ chasm.NexusCompletionHandler = &Scheduler{} @@ -127,6 +128,15 @@ func (s *Scheduler) HandleNexusCompletion( ) return nil } + if !internal.TracksCompletionResult(start.GetOverlapPolicy()) && start.GetRunId() == "" { + // A callback can beat the start RPC response. Preserve the pending entry + // until that response records its identity and action count. + metricsHandler.Counter(metrics.ScheduleCallbackIgnored.Name()).Record( + 1, + metrics.ReasonTag(callbackIgnoredStartNotRecorded), + ) + return nil + } if start.GetCompleted() != nil { // Completion callbacks may be validly redelivered, for example after a workflow reset. // Preserve state but keep the duplicate observable through the log and metric. diff --git a/chasm/lib/scheduler/gen/schedulerpb/v1/message.pb.go b/chasm/lib/scheduler/gen/schedulerpb/v1/message.pb.go index ed347794d8d..2e7c85789e4 100644 --- a/chasm/lib/scheduler/gen/schedulerpb/v1/message.pb.go +++ b/chasm/lib/scheduler/gen/schedulerpb/v1/message.pb.go @@ -188,8 +188,13 @@ type WorkflowMigrationState struct { PreMigrationPaused bool `protobuf:"varint,1,opt,name=pre_migration_paused,json=preMigrationPaused,proto3" json:"pre_migration_paused,omitempty"` // The schedule's notes before migration was initiated. PreMigrationNotes string `protobuf:"bytes,2,opt,name=pre_migration_notes,json=preMigrationNotes,proto3" json:"pre_migration_notes,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Request ID used for every StartWorkflowExecution attempt of the destination + // V1 workflow. It identifies the destination as created by this migration. + // Empty only on migrations persisted before this field existed; the migrate + // task assigns one before its first start attempt. + RequestId string `protobuf:"bytes,3,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *WorkflowMigrationState) Reset() { @@ -236,6 +241,13 @@ func (x *WorkflowMigrationState) GetPreMigrationNotes() string { return "" } +func (x *WorkflowMigrationState) GetRequestId() string { + if x != nil { + return x.RequestId + } + return "" +} + // CHASM scheduler's Generator internal state. type GeneratorState struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -774,10 +786,12 @@ const file_temporal_server_chasm_lib_scheduler_proto_v1_message_proto_rawDesc = " \x01(\bR\bsentinel\x12s\n" + "\x12workflow_migration\x18\v \x01(\v2D.temporal.server.chasm.lib.scheduler.proto.v1.WorkflowMigrationStateR\x11workflowMigration\x12B\n" + "\x0fidle_close_time\x18\f \x01(\v2\x1a.google.protobuf.TimestampR\ridleCloseTime\x12B\n" + - "\x0flast_event_time\x18\r \x01(\v2\x1a.google.protobuf.TimestampR\rlastEventTime\"z\n" + + "\x0flast_event_time\x18\r \x01(\v2\x1a.google.protobuf.TimestampR\rlastEventTime\"\x99\x01\n" + "\x16WorkflowMigrationState\x120\n" + "\x14pre_migration_paused\x18\x01 \x01(\bR\x12preMigrationPaused\x12.\n" + - "\x13pre_migration_notes\x18\x02 \x01(\tR\x11preMigrationNotes\"\xfd\x01\n" + + "\x13pre_migration_notes\x18\x02 \x01(\tR\x11preMigrationNotes\x12\x1d\n" + + "\n" + + "request_id\x18\x03 \x01(\tR\trequestId\"\xfd\x01\n" + "\x0eGeneratorState\x12J\n" + "\x13last_processed_time\x18\x03 \x01(\v2\x1a.google.protobuf.TimestampR\x11lastProcessedTime\x12J\n" + "\x13future_action_times\x18\x04 \x03(\v2\x1a.google.protobuf.TimestampR\x11futureActionTimes\x12S\n" + diff --git a/chasm/lib/scheduler/generator_killswitch_test.go b/chasm/lib/scheduler/generator_killswitch_test.go index 9fbd7fceebd..dcafd1fbed9 100644 --- a/chasm/lib/scheduler/generator_killswitch_test.go +++ b/chasm/lib/scheduler/generator_killswitch_test.go @@ -5,6 +5,7 @@ import ( "time" "github.com/stretchr/testify/require" + enumspb "go.temporal.io/api/enums/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/lib/scheduler" "go.temporal.io/server/common/clock" @@ -42,6 +43,7 @@ func TestGeneratorJitterFixKillswitch(t *testing.T) { schedule := defaultSchedule() schedule.Spec.Interval[0].Interval = durationpb.New(15 * time.Minute) schedule.Spec.Jitter = durationpb.New(10 * time.Minute) + schedule.Policies.OverlapPolicy = enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ALL env := newSchedulerTestEngine(t, schedule, withEngineTimeSource(ts), withEngineSpecProcessor(processor)) // Change the setting after creation to exercise existing schedules. diff --git a/chasm/lib/scheduler/generator_wakeup_test.go b/chasm/lib/scheduler/generator_wakeup_test.go index c895cbb0a12..b35cbd75f03 100644 --- a/chasm/lib/scheduler/generator_wakeup_test.go +++ b/chasm/lib/scheduler/generator_wakeup_test.go @@ -54,7 +54,7 @@ func TestGeneratorNextWakeupTimeStaysAtPendingJitteredFire(t *testing.T) { require.NoError(t, err) var timerDeadlines []time.Time observeTimers := func(config *engineTestConfig) { - config.engineOpts = append(config.engineOpts, chasmtest.WithInvariantCheck(func(_ *testing.T, node *chasm.Node, _ chasm.RootComponent) { + config.engineOpts = append(config.engineOpts, chasmtest.WithInvariantCheck(func(_ chasmtest.TB, node *chasm.Node, _ chasm.RootComponent) { timerDeadlines = nil generator := node.Snapshot(nil).Nodes[generatorPath] for _, task := range generator.GetMetadata().GetComponentAttributes().GetPureTasks() { diff --git a/chasm/lib/scheduler/handler_test.go b/chasm/lib/scheduler/handler_test.go index 27bab78c1c7..a03ad14c5dc 100644 --- a/chasm/lib/scheduler/handler_test.go +++ b/chasm/lib/scheduler/handler_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/stretchr/testify/require" + enumspb "go.temporal.io/api/enums/v1" schedulepb "go.temporal.io/api/schedule/v1" "go.temporal.io/api/serviceerror" "go.temporal.io/api/workflowservice/v1" @@ -196,7 +197,9 @@ func TestHandler_MigrateToWorkflow_Sentinel(t *testing.T) { } func TestHandler_PatchSchedule_DedupsRequestID(t *testing.T) { - e := newSchedulerTestEngine(t, defaultSchedule()) + schedule := defaultSchedule() + schedule.Policies.OverlapPolicy = enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ALL + e := newSchedulerTestEngine(t, schedule) h := scheduler.NewTestHandler(e.logger) patch := func(requestID string) { _, err := h.PatchSchedule(e.engineCtx, &schedulerpb.PatchScheduleRequest{ diff --git a/chasm/lib/scheduler/helper_test.go b/chasm/lib/scheduler/helper_test.go index b73ed03e28f..f64ea8aeacb 100644 --- a/chasm/lib/scheduler/helper_test.go +++ b/chasm/lib/scheduler/helper_test.go @@ -7,15 +7,14 @@ import ( "time" "github.com/stretchr/testify/require" - commonpb "go.temporal.io/api/common/v1" schedulepb "go.temporal.io/api/schedule/v1" - workflowpb "go.temporal.io/api/workflow/v1" "go.temporal.io/api/workflowservice/v1" persistencespb "go.temporal.io/server/api/persistence/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/chasmtest" "go.temporal.io/server/chasm/lib/scheduler" "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" + "go.temporal.io/server/chasm/lib/scheduler/schedulertest" "go.temporal.io/server/common/backoff" "go.temporal.io/server/common/clock" "go.temporal.io/server/common/log" @@ -25,115 +24,48 @@ import ( "go.temporal.io/server/service/history/tasks" legacyscheduler "go.temporal.io/server/service/worker/scheduler" "go.uber.org/mock/gomock" - "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/timestamppb" ) const ( - namespace = "ns" - namespaceID = "ns-id" - scheduleID = "sched-id" + namespace = schedulertest.Namespace + namespaceID = schedulertest.NamespaceID + scheduleID = schedulertest.ScheduleID - defaultInterval = 1 * time.Minute - defaultCatchupWindow = 5 * time.Minute + defaultInterval = schedulertest.DefaultInterval + defaultCatchupWindow = schedulertest.DefaultCatchupWindow ) // newLegacySpecBuilder builds a legacy SpecBuilder with the given warn/max compute-limit bounds. // A value of 0 means "use the default" (GetNextTime treats a non-positive bound as its default). func newLegacySpecBuilder(warnIter, maxIter int) *legacyscheduler.SpecBuilder { - return legacyscheduler.NewSpecBuilder(func() int { return warnIter }, func() int { return maxIter }) + return schedulertest.NewSpecBuilder(warnIter, maxIter) } // defaultSchedule returns a protobuf definition for a schedule matching this // package's other testing defaults. func defaultSchedule() *schedulepb.Schedule { - return &schedulepb.Schedule{ - Spec: &schedulepb.ScheduleSpec{ - Interval: []*schedulepb.IntervalSpec{ - { - Interval: durationpb.New(defaultInterval), - Phase: durationpb.New(0), - }, - }, - }, - Action: &schedulepb.ScheduleAction{ - Action: &schedulepb.ScheduleAction_StartWorkflow{ - StartWorkflow: &workflowpb.NewWorkflowExecutionInfo{ - WorkflowId: "scheduled-wf", - WorkflowType: &commonpb.WorkflowType{Name: "scheduled-wf-type"}, - }, - }, - }, - Policies: &schedulepb.SchedulePolicies{ - CatchupWindow: durationpb.New(defaultCatchupWindow), - }, - State: &schedulepb.ScheduleState{ - Paused: false, - LimitedActions: false, - RemainingActions: 0, - }, - } + return schedulertest.DefaultSchedule() } +// defaultConfig is the shared default config with the production-style jittered +// exponential retry policy. The lifecycle harness swaps in a deterministic one +// so that rapid replays a failure identically; these tests never needed that. func defaultConfig() *scheduler.Config { - return &scheduler.Config{ - Tweakables: func(_ string) scheduler.Tweakables { - return scheduler.DefaultTweakables - }, - ServiceCallTimeout: func() time.Duration { - return 5 * time.Second - }, - EncodeInternalTokenWithEnvelope: func(string) bool { - return true - }, - RetryPolicy: func() backoff.RetryPolicy { - return backoff.NewExponentialRetryPolicy(1 * time.Second) - }, + cfg := schedulertest.DefaultConfig() + cfg.RetryPolicy = func() backoff.RetryPolicy { + return backoff.NewExponentialRetryPolicy(1 * time.Second) } + return cfg } func newTestLibrary(logger log.Logger, specProcessor scheduler.SpecProcessor) *scheduler.Library { - config := defaultConfig() - specBuilder := newLegacySpecBuilder(0, 0) - invokerOpts := scheduler.InvokerTaskHandlerOptions{ - Config: config, - MetricsHandler: metrics.NoopMetricsHandler, - BaseLogger: logger, - SpecProcessor: specProcessor, - } - return scheduler.NewLibrary( - config, - nil, - scheduler.NewSchedulerIdleTaskHandler(scheduler.SchedulerIdleTaskHandlerOptions{ - Config: config, - MetricsHandler: metrics.NoopMetricsHandler, - BaseLogger: logger, - }), - scheduler.NewSchedulerCallbacksTaskHandler(scheduler.SchedulerCallbacksTaskHandlerOptions{ - Config: config, - MetricsHandler: metrics.NoopMetricsHandler, - }), - scheduler.NewGeneratorTaskHandler(scheduler.GeneratorTaskHandlerOptions{ - Config: config, - MetricsHandler: metrics.NoopMetricsHandler, - BaseLogger: logger, - SpecProcessor: specProcessor, - SpecBuilder: specBuilder, - }), - scheduler.NewInvokerExecuteTaskHandler(invokerOpts), - scheduler.NewInvokerProcessBufferTaskHandler(invokerOpts), - scheduler.NewBackfillerTaskHandler(scheduler.BackfillerTaskHandlerOptions{ - Config: config, - MetricsHandler: metrics.NoopMetricsHandler, - BaseLogger: logger, - SpecProcessor: specProcessor, - }), - scheduler.NewSchedulerMigrateToWorkflowTaskHandler(scheduler.SchedulerMigrateToWorkflowTaskHandlerOptions{ - Config: config, - MetricsHandler: metrics.NoopMetricsHandler, - BaseLogger: logger, - }), - ) + return schedulertest.NewTestLibrary(schedulertest.LibraryOptions{ + Config: defaultConfig(), + Logger: logger, + SpecProcessor: specProcessor, + SpecBuilder: newLegacySpecBuilder(0, 0), + }) } // testEnv holds all components needed for scheduler tests. @@ -149,9 +81,9 @@ type testEnv struct { MockEngine *chasm.MockEngine Logger log.Logger - // allowStuckReason, when non-empty, suppresses the stuckness invariant - // asserted by CloseTransaction. See AllowStuck. - allowStuckReason string + // state is the shared stuck and idle-deadline invariant asserted by + // CloseTransaction. See AllowStuck. + state *schedulertest.StateInvariant } // testEnvConfig holds configuration options for testEnv. @@ -201,17 +133,7 @@ func expiredSchedule(now time.Time) *schedulepb.Schedule { // newRealSpecProcessor creates a real SpecProcessor for tests. func newRealSpecProcessor(ctrl *gomock.Controller, logger log.Logger) scheduler.SpecProcessor { - mockMetrics := metrics.NewMockHandler(ctrl) - mockMetrics.EXPECT().Counter(gomock.Any()).Return(metrics.NoopCounterMetricFunc).AnyTimes() - mockMetrics.EXPECT().WithTags(gomock.Any()).Return(mockMetrics).AnyTimes() - mockMetrics.EXPECT().Timer(gomock.Any()).Return(metrics.NoopTimerMetricFunc).AnyTimes() - - return scheduler.NewSpecProcessor( - defaultConfig(), - mockMetrics, - logger, - newLegacySpecBuilder(0, 0), - ) + return schedulertest.NewSpecProcessor(ctrl, logger, newLegacySpecBuilder(0, 0)) } // engineTestConfig holds configuration options for newTestEngineContext. @@ -237,7 +159,6 @@ func withEngineSpecProcessor(sp scheduler.SpecProcessor) engineTestOption { func withEngineTimeSource(ts *clock.EventTimeSource) engineTestOption { return func(c *engineTestConfig) { c.timeSource = ts - c.engineOpts = append(c.engineOpts, chasmtest.WithTimeSource(ts)) } } @@ -268,28 +189,34 @@ func newTestEngineContextFromConfig( logger log.Logger, config *engineTestConfig, ) (*chasmtest.Engine, context.Context) { - specProcessor := config.specProcessor - if specProcessor == nil { - specProcessor = newRealSpecProcessor(gomock.NewController(t), logger) - } - - registry := chasm.NewRegistry(logger) - require.NoError(t, registry.Register(&chasm.CoreLibrary{})) - require.NoError(t, registry.Register(newTestLibrary(logger, specProcessor))) + base := newBaseEngine(t, logger, config) + return base.Engine, base.Ctx +} - config.engineOpts = append(config.engineOpts, chasmtest.WithInvariantCheck( - func(t *testing.T, node *chasm.Node, root chasm.RootComponent) { - requireValidSchedulerState(t, registry, node, root) - }, - )) - engine := chasmtest.NewEngine(t, registry, config.engineOpts...) - return engine, chasm.NewEngineContext(context.Background(), engine) +// newBaseEngine builds the shared scheduler engine with this package's config, +// and spec builder. The engine asserts the shared stuck and idle-deadline +// invariants after every transition. +func newBaseEngine(t *testing.T, logger log.Logger, config *engineTestConfig) *schedulertest.SchedulerEngine { + opts := []schedulertest.Option{ + schedulertest.WithLogger(logger), + schedulertest.WithSchedulerConfig(defaultConfig()), + schedulertest.WithSpecBuilder(newLegacySpecBuilder(0, 0)), + schedulertest.WithEngineOptions(config.engineOpts...), + } + if config.specProcessor != nil { + opts = append(opts, schedulertest.WithSpecProcessor(config.specProcessor)) + } + if config.timeSource != nil { + opts = append(opts, schedulertest.WithTimeSource(config.timeSource)) + } + return schedulertest.NewEngine(t, opts...) } // schedulerTestEngine bundles a CHASM test engine, a created Scheduler root // component, and convenience accessors, for tests that drive a Scheduler // through its actual task handlers rather than the newTestEnv rapid harness. type schedulerTestEngine struct { + base *schedulertest.SchedulerEngine engine *chasmtest.Engine engineCtx context.Context rootRef chasm.ComponentRef @@ -313,14 +240,9 @@ func newSchedulerTestEngine( if config.timeSource == nil { config.timeSource = clock.NewEventTimeSource() config.timeSource.Update(time.Now()) - config.engineOpts = append(config.engineOpts, chasmtest.WithTimeSource(config.timeSource)) } - engine, engineCtx := newTestEngineContextFromConfig(t, logger, config) - rootRef := chasm.NewComponentRef[*scheduler.Scheduler](chasm.ExecutionKey{ - NamespaceID: namespaceID, - BusinessID: scheduleID, - }) - _, err := scheduler.NewTestHandler(logger).CreateSchedule(engineCtx, &schedulerpb.CreateScheduleRequest{ + base := newBaseEngine(t, logger, config) + _, err := scheduler.NewTestHandler(logger).CreateSchedule(base.Ctx, &schedulerpb.CreateScheduleRequest{ NamespaceId: namespaceID, FrontendRequest: &workflowservice.CreateScheduleRequest{ Namespace: namespace, @@ -331,9 +253,10 @@ func newSchedulerTestEngine( }) require.NoError(t, err) return &schedulerTestEngine{ - engine: engine, - engineCtx: engineCtx, - rootRef: rootRef, + base: base, + engine: base.Engine, + engineCtx: base.Ctx, + rootRef: base.Ref, logger: logger, timeSource: config.timeSource, } @@ -344,15 +267,7 @@ func newSchedulerTestEngine( func (e *schedulerTestEngine) updateScheduler( update func(*scheduler.Scheduler, chasm.MutableContext) error, ) error { - _, _, err := chasm.UpdateComponent( - e.engineCtx, - e.rootRef, - func(s *scheduler.Scheduler, ctx chasm.MutableContext, _ struct{}) (struct{}, error) { - return struct{}{}, update(s, ctx) - }, - struct{}{}, - ) - return err + return e.base.Update(update) } // readScheduler runs read against the Scheduler root component through the @@ -360,15 +275,7 @@ func (e *schedulerTestEngine) updateScheduler( func (e *schedulerTestEngine) readScheduler( read func(*scheduler.Scheduler, chasm.Context) error, ) error { - _, err := chasm.ReadComponent( - e.engineCtx, - e.rootRef, - func(s *scheduler.Scheduler, ctx chasm.Context, _ struct{}) (struct{}, error) { - return struct{}{}, read(s, ctx) - }, - struct{}{}, - ) - return err + return e.base.Read(read) } // newTestEnv creates a new test environment with the given options. @@ -452,6 +359,7 @@ func newTestEnv(t *testing.T, opts ...testEnvOption) *testEnv { Scheduler: sched, SpecProcessor: specProcessor, Logger: logger, + state: schedulertest.NewStateInvariant(registry, defaultConfig(), logger), } if config.withMockEngine { @@ -489,10 +397,7 @@ func (e *testEnv) CloseTransaction() error { if err != nil { return err } - if e.allowStuckReason == "" { - requireNotStuck(e.t, e.Node, e.Scheduler) - requireIdleCloseTimeBacked(e.t, e.Registry, e.Node, e.Scheduler) - } + e.state.Check(e.t, e.Node, e.Scheduler) return nil } @@ -505,10 +410,7 @@ func (e *testEnv) CloseTransaction() error { // finding worth reviewing. func (e *testEnv) AllowStuck(reason string) { e.t.Helper() - if reason == "" { - e.t.Fatal("AllowStuck requires a reason explaining why this test tolerates a stuck scheduler") - } - e.allowStuckReason = reason + e.state.AllowStuck(e.t, reason) } // HasTask returns true if the given task type was added with the given visibilityTime. diff --git a/chasm/lib/scheduler/invariant_test.go b/chasm/lib/scheduler/invariant_test.go deleted file mode 100644 index b35b08be0f2..00000000000 --- a/chasm/lib/scheduler/invariant_test.go +++ /dev/null @@ -1,270 +0,0 @@ -package scheduler_test - -import ( - "fmt" - "slices" - "strings" - "testing" - "time" - - "github.com/stretchr/testify/require" - "go.temporal.io/server/chasm" - "go.temporal.io/server/chasm/lib/scheduler" - "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" - "google.golang.org/protobuf/types/known/timestamppb" -) - -// This file implements the "stuckness" invariant for Schedules/V2. -// -// Production has repeatedly produced schedules that are open, unpaused, and -// carry no pending work: they never re-arm and never close. Every confirmed -// instance violated exactly one predicate: -// -// !sched.Closed => the tree carries at least one live logical task. -// -// Note that this is a liveness property, not a transition-validity property. -// The scheduler has no state enum to validate transitions against: -// Scheduler.LifecycleState is derived and two-valued (Completed iff Closed), -// and every sub-component hardcodes LifecycleStateRunning. In each known bug -// every transition taken was legal; the failure was that no transition -// happened at all. -// -// IMPORTANT: nothing here may touch live components. Materializing a chasm -// Field (e.g. sched.Invoker.Get(ctx)) escalates the root node's valueState and -// marks the tree dirty even for a read, which is itself one of the fault -// mechanisms behind the production bugs. So this check reads only: -// - plain Go field access on the embedded *schedulerpb.SchedulerState proto -// - the serialized snapshot returned by Node.Snapshot -// It never constructs a chasm.Context. - -// schedState is a test-only vocabulary for the scheduler's lifecycle. It is -// derived, never persisted: there is deliberately no state enum on the -// component, and adding one is a proto change with V1->V2 migration and -// rollback implications that this check does not require. -type schedState string - -const ( - // stateSentinel is a schedule-ID reservation. Inert by design. - stateSentinel schedState = "sentinel" - // stateMigrating is mid-migration to a workflow-backed (V1) scheduler, - // driven by SchedulerMigrateToWorkflowTask. - stateMigrating schedState = "migrating" - // stateCompleted is terminal: Closed is set. - stateCompleted schedState = "completed" - // stateIdle is non-terminal and quiescent, awaiting its idle timer. - stateIdle schedState = "idle" - // stateRunning is non-terminal with pending work armed. - stateRunning schedState = "running" - // stateStuck is the bug: non-terminal, but nothing will ever wake it. - stateStuck schedState = "stuck" -) - -// liveTaskCount counts logical tasks across every node in the tree. -// -// This deliberately counts *logical* tasks, from ChasmComponentAttributes. -// PureTasks and SideEffectTasks, rather than physical tasks. A physical task -// can exist on the backend while the logical task it points at has been reaped -// or invalidated, so counting physical tasks reports a schedule as armed when -// it is in fact stuck. Side-effect tasks count because some valid states, such -// as migration callback attachment, rely on one before arming pure tasks. -// (chasmtest.Engine.Tasks and testEnv.NodeBackend.TasksByCategory both expose -// physical tasks, and are the wrong source for this check.) -// -// node must be the root and the tree must be clean; Node.Snapshot panics -// otherwise. Both hold immediately after a successful CloseTransaction. -func liveTaskCount(node *chasm.Node) (int, map[string]int) { - byPath := make(map[string]int) - total := 0 - for path, n := range node.Snapshot(nil).Nodes { - attrs := n.GetMetadata().GetComponentAttributes() - taskCount := len(attrs.GetPureTasks()) + len(attrs.GetSideEffectTasks()) - if taskCount == 0 { - continue - } - byPath[path] = taskCount - total += taskCount - } - return total, byPath -} - -// derivedState computes the scheduler's lifecycle state from the tuple that -// actually represents it: (Sentinel, WorkflowMigration, Closed, IdleCloseTime, -// live logical task count). -func derivedState(sched *scheduler.Scheduler, liveTasks int) schedState { - switch { - case sched.GetSentinel(): - return stateSentinel - case sched.GetWorkflowMigration() != nil: - return stateMigrating - case sched.GetClosed(): - return stateCompleted - case liveTasks == 0: - return stateStuck - case sched.GetIdleCloseTime() != nil: - return stateIdle - default: - return stateRunning - } -} - -// idleTaskCount counts armed SchedulerIdleTasks across the tree, resolving the -// task's registered type ID rather than matching on node path (the root -// Scheduler component also carries callback and migration tasks). -func idleTaskCount(registry *chasm.Registry, node *chasm.Node) int { - wantTypeID, ok := registry.TaskIDFor(&schedulerpb.SchedulerIdleTask{}) - if !ok { - return -1 - } - count := 0 - for _, n := range node.Snapshot(nil).Nodes { - for _, task := range n.GetMetadata().GetComponentAttributes().GetPureTasks() { - if task.GetTypeId() == wantTypeID { - count++ - } - } - } - return count -} - -// requireIdleCloseTimeBacked asserts that a declared idle deadline is actually -// backed by an armed idle task. -// -// IdleCloseTime is surfaced as the ScheduleIdleCloseTime search attribute and -// is what the production stuck-schedule scanner reads. When -// SchedulerIdleTaskHandler.Validate drops a task as expiration_shift and -// nothing arms a replacement, IdleCloseTime is left behind: the schedule never -// closes, and the very signal built to detect that reports a close time that -// will never arrive. That combination is always wrong. -// -// Only asserted in this direction. Sentinels legitimately arm an idle task -// without setting IdleCloseTime (see NewSentinel), so an armed task with no -// declared deadline is not a defect. -func requireIdleCloseTimeBacked(t *testing.T, registry *chasm.Registry, node *chasm.Node, sched *scheduler.Scheduler) { - t.Helper() - - if sched.GetSentinel() || sched.GetClosed() || sched.GetIdleCloseTime() == nil { - return - } - total, byPath := liveTaskCount(node) - require.NotZero(t, idleTaskCount(registry, node), - "IdleCloseTime is set to %v but no SchedulerIdleTask is armed, so the schedule will "+ - "never close and ScheduleIdleCloseTime reports a deadline that will never arrive: %s", - sched.GetIdleCloseTime().AsTime(), describeSched(sched, total, byPath)) -} - -// requireNotStuck asserts the stuckness invariant against a clean tree. -func requireNotStuck(t *testing.T, node *chasm.Node, sched *scheduler.Scheduler) { - t.Helper() - - total, byPath := liveTaskCount(node) - state := derivedState(sched, total) - require.NotEqual(t, stateStuck, state, "scheduler is stuck: %s", describeSched(sched, total, byPath)) -} - -func requireValidSchedulerState( - t *testing.T, - registry *chasm.Registry, - node *chasm.Node, - root chasm.RootComponent, -) { - t.Helper() - - sched, ok := root.(*scheduler.Scheduler) - require.Truef(t, ok, "scheduler test engine root has unexpected type %T", root) - requireNotStuck(t, node, sched) - requireIdleCloseTimeBacked(t, registry, node, sched) -} - -// describeSched renders the full state tuple for a failure message, so a -// triage does not require re-running under a debugger. -func describeSched(sched *scheduler.Scheduler, total int, byPath map[string]int) string { - paths := make([]string, 0, len(byPath)) - for p := range byPath { - paths = append(paths, fmt.Sprintf("%s=%d", p, byPath[p])) - } - slices.Sort(paths) - - var b strings.Builder - fmt.Fprintf(&b, "\n scheduleID: %s", sched.GetScheduleId()) - fmt.Fprintf(&b, "\n closed: %v", sched.GetClosed()) - fmt.Fprintf(&b, "\n sentinel: %v", sched.GetSentinel()) - fmt.Fprintf(&b, "\n paused: %v", sched.GetSchedule().GetState().GetPaused()) - fmt.Fprintf(&b, "\n limitedActions: %v", sched.GetSchedule().GetState().GetLimitedActions()) - fmt.Fprintf(&b, "\n remaining: %d", sched.GetSchedule().GetState().GetRemainingActions()) - fmt.Fprintf(&b, "\n conflictToken: %d", sched.GetConflictToken()) - fmt.Fprintf(&b, "\n idleCloseTime: %v", sched.GetIdleCloseTime().AsTime()) - fmt.Fprintf(&b, "\n migration: %v", sched.GetWorkflowMigration() != nil) - fmt.Fprintf(&b, "\n liveTasks: %d [%s]", total, strings.Join(paths, " ")) - return b.String() -} - -// TestDerivedState pins the classification tuple. Without this, a refactor -// that made derivedState never return stateStuck would silently disarm every -// stuckness assertion in the package while leaving the suite green. -func TestDerivedState(t *testing.T) { - t.Parallel() - - cases := []struct { - name string - state *schedulerpb.SchedulerState - liveTasks int - want schedState - }{ - { - name: "open with an armed task is running", - state: &schedulerpb.SchedulerState{}, - liveTasks: 1, - want: stateRunning, - }, - { - name: "open with an armed task and an idle deadline is idle", - state: &schedulerpb.SchedulerState{IdleCloseTime: timestamppb.New(time.Unix(100, 0))}, - liveTasks: 1, - want: stateIdle, - }, - { - name: "open with no armed task is stuck", - state: &schedulerpb.SchedulerState{}, - liveTasks: 0, - want: stateStuck, - }, - { - name: "an idle deadline does not excuse a missing task", - state: &schedulerpb.SchedulerState{IdleCloseTime: timestamppb.New(time.Unix(100, 0))}, - liveTasks: 0, - want: stateStuck, - }, - { - name: "closed with no armed task is terminal, not stuck", - state: &schedulerpb.SchedulerState{Closed: true}, - liveTasks: 0, - want: stateCompleted, - }, - { - name: "sentinel is inert by design", - state: &schedulerpb.SchedulerState{Sentinel: true}, - liveTasks: 0, - want: stateSentinel, - }, - { - name: "mid-migration is excused", - state: &schedulerpb.SchedulerState{WorkflowMigration: &schedulerpb.WorkflowMigrationState{}}, - liveTasks: 0, - want: stateMigrating, - }, - { - name: "closed takes precedence over a stale idle deadline", - state: &schedulerpb.SchedulerState{Closed: true, IdleCloseTime: timestamppb.New(time.Unix(100, 0))}, - liveTasks: 0, - want: stateCompleted, - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - sched := &scheduler.Scheduler{SchedulerState: tc.state} - require.Equal(t, tc.want, derivedState(sched, tc.liveTasks)) - }) - } -} diff --git a/chasm/lib/scheduler/invoker.go b/chasm/lib/scheduler/invoker.go index 3580a43d2ae..45ea86b4011 100644 --- a/chasm/lib/scheduler/invoker.go +++ b/chasm/lib/scheduler/invoker.go @@ -28,6 +28,20 @@ type Invoker struct { EventLog chasm.Field[*EventLog] } +// canMigrateToWorkflow reports whether V1 can take over this invoker's work. +// V1 state has no place for pending cancel/terminate requests or for starts that +// are ready or retrying, so those would be lost on migration. A start with +// Attempt > 0 becomes transferable once it records a RunId or a Completed +// result, or once processing resets its Attempt to 0 or -1. +func (i *Invoker) canMigrateToWorkflow() bool { + if len(i.CancelWorkflows) > 0 || len(i.TerminateWorkflows) > 0 { + return false + } + return !slices.ContainsFunc(i.BufferedStarts, func(start *schedulespb.BufferedStart) bool { + return start.GetAttempt() > 0 && start.GetRunId() == "" && start.GetCompleted() == nil + }) +} + func (i *Invoker) LifecycleState(ctx chasm.Context) chasm.LifecycleState { return chasm.LifecycleStateRunning } @@ -123,6 +137,18 @@ func (i *Invoker) recordProcessBufferResult(ctx chasm.MutableContext, result *pr i.TerminateWorkflows = append(i.GetTerminateWorkflows(), result.terminateWorkflows...) i.LastProcessedTime = timestamppb.New(ctx.Now(i)) + // A discarded selection cannot produce the completion that would release + // deferred work. Reconsider it while each pass removes at least one start. + if len(discards) > 0 && !slices.ContainsFunc(starts, func(start *schedulespb.BufferedStart) bool { + return start.GetCompleted() == nil && (start.GetRunId() != "" || start.GetAttempt() > 0) + }) { + for _, start := range starts { + if start.GetAttempt() == -1 { + start.Attempt = 0 + } + } + } + // Re-arm tasks if this call changed state, or if the LastProcessedTime advance // just unblocked backed-off starts. i.addTasks(ctx) @@ -197,6 +223,7 @@ func (i *Invoker) recordExecuteResult( } return failed }) + dropped := removedStarts > 0 i.CancelWorkflows = slices.DeleteFunc(i.GetCancelWorkflows(), func(we *commonpb.WorkflowExecution) bool { canceled := canceled[we.RunId] if canceled { @@ -243,6 +270,12 @@ func (i *Invoker) recordExecuteResult( _, remove := startedUntracked[start.GetRequestId()] return remove }) + // A selected start holds the overlap slot until its RunId is recorded. Dropping it + // produces no completion, and recording its RunId makes it cancelable, so either + // way deferred starts must be re-evaluated against the new state. + if dropped || newlyStarted > 0 { + i.reenableDeferredStarts() + } i.getOrCreateEventLog(ctx).LogEvent(ctx, fmt.Sprintf("recordExecuteResult kicked off %d starts, removed %d starts, retried %d starts", newlyStarted, @@ -304,11 +337,7 @@ func (i *Invoker) recordCompletedAction( // Re-enable deferred starts (Attempt == -1) so they can be re-processed by // ProcessBuffer now that a workflow has completed. This allows the overlap // policy to be re-evaluated. - for _, start := range i.BufferedStarts { - if start.Attempt == -1 { - start.Attempt = 0 - } - } + i.reenableDeferredStarts() // Update DesiredTime on the first pending start for metrics. DesiredTime is used // to drive action latency between buffered starts (the time it takes between @@ -333,6 +362,16 @@ func (i *Invoker) recordCompletedAction( return } +// reenableDeferredStarts returns deferred starts (Attempt == -1) to the pending +// state so the next ProcessBuffer pass re-evaluates their overlap policy. +func (i *Invoker) reenableDeferredStarts() { + for _, start := range i.BufferedStarts { + if start.Attempt == -1 { + start.Attempt = 0 + } + } +} + // addTasks adds both ProcessBuffer and Execute tasks as needed. It should be // called when completing processing/executing tasks, to drive backoff/retry. func (i *Invoker) addTasks(ctx chasm.MutableContext) { diff --git a/chasm/lib/scheduler/invoker_tasks.go b/chasm/lib/scheduler/invoker_tasks.go index bb9234e77b2..f32d66d7337 100644 --- a/chasm/lib/scheduler/invoker_tasks.go +++ b/chasm/lib/scheduler/invoker_tasks.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "slices" "sync" "time" @@ -18,6 +19,7 @@ import ( "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" "go.temporal.io/server/chasm/lib/scheduler/internal" "go.temporal.io/server/common" + "go.temporal.io/server/common/clock" "go.temporal.io/server/common/log" "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/metrics" @@ -44,6 +46,9 @@ type ( // the request makes it through metering's interceptor. Because we don't change for // terminate/cancels, we can go directly to history for other service calls. FrontendClient workflowservice.WorkflowServiceClient + + // TimeSource stamps StartTime on started workflows. Defaults to real time. + TimeSource clock.TimeSource `optional:"true"` } InvokerExecuteTaskHandler struct { @@ -53,6 +58,7 @@ type ( baseLogger log.Logger historyClient resource.HistoryClient frontendClient workflowservice.WorkflowServiceClient + timeSource clock.TimeSource } InvokerProcessBufferTaskHandler struct { @@ -108,12 +114,17 @@ const ( ) func NewInvokerExecuteTaskHandler(opts InvokerTaskHandlerOptions) *InvokerExecuteTaskHandler { + timeSource := opts.TimeSource + if timeSource == nil { + timeSource = clock.NewRealTimeSource() + } return &InvokerExecuteTaskHandler{ config: opts.Config, metricsHandler: opts.MetricsHandler, baseLogger: opts.BaseLogger, historyClient: opts.HistoryClient, frontendClient: opts.FrontendClient, + timeSource: timeSource, } } @@ -234,7 +245,8 @@ func (h *InvokerExecuteTaskHandler) Execute( // Invoker will never have work pending for more than one of these calls (terminate, // cancel, start) at a time, so it isn't sensible to run them in parallel. The // structure below is simply for code simplicity. - ictx := h.newInvokerTaskHandlerContext(ctx, scheduler) + // One budget is shared by all three phases, so pass the context by pointer. + ictx := new(h.newInvokerTaskHandlerContext(ctx, scheduler)) result = result.Append(h.terminateWorkflows(ictx, logger, metricsHandler, scheduler, invoker.GetTerminateWorkflows())) result = result.Append(h.cancelWorkflows(ictx, logger, metricsHandler, scheduler, invoker.GetCancelWorkflows())) result = result.Append(h.startWorkflows( @@ -289,7 +301,7 @@ func (i *invokerTaskHandlerContext) takeNextAction() bool { // cancelWorkflows does a best-effort attempt to cancel all workflow executions provided in targets. func (h *InvokerExecuteTaskHandler) cancelWorkflows( - ctx invokerTaskHandlerContext, + ctx *invokerTaskHandlerContext, logger log.Logger, metricsHandler metrics.Handler, scheduler *Scheduler, @@ -331,7 +343,7 @@ func (h *InvokerExecuteTaskHandler) cancelWorkflows( // terminateWorkflows does a best-effort attempt to terminate all workflow executions provided in targets. func (h *InvokerExecuteTaskHandler) terminateWorkflows( - ctx invokerTaskHandlerContext, + ctx *invokerTaskHandlerContext, logger log.Logger, metricsHandler metrics.Handler, scheduler *Scheduler, @@ -373,7 +385,7 @@ func (h *InvokerExecuteTaskHandler) terminateWorkflows( // startWorkflows executes the provided list of starts, returning a result with their outcomes. func (h *InvokerExecuteTaskHandler) startWorkflows( - ctx invokerTaskHandlerContext, + ctx *invokerTaskHandlerContext, logger log.Logger, metricsHandler metrics.Handler, scheduler *Scheduler, @@ -527,7 +539,14 @@ func (h *InvokerProcessBufferTaskHandler) processBuffer( scheduler *Scheduler, ) (result processBufferResult) { runningWorkflows := invoker.runningWorkflowExecutions() - isRunning := len(runningWorkflows) > 0 + // A start that was selected for execution but has no recorded RunId yet still + // holds the overlap slot, otherwise a second pass could select another start + // before the first StartWorkflowExecution response is recorded. + isRunning := len(runningWorkflows) > 0 || + slices.ContainsFunc(invoker.GetBufferedStarts(), func(start *schedulespb.BufferedStart) bool { + return start.GetAttempt() > 0 && start.GetCompleted() == nil && + internal.TracksCompletionResult(start.GetOverlapPolicy()) + }) result.missedCatchupByActionRunning = make(map[bool]int64) // Processing ignores starts that are already executing or backing off. An existing @@ -741,7 +760,7 @@ func (h *InvokerExecuteTaskHandler) startWorkflow( if err != nil { return err } - actualStartTime := time.Now() + actualStartTime := h.timeSource.Now() // Reconstruct virtual time from wall time. actualStartTime = actualStartTime.Add( timeSkippingStatePropagation.GetInitialSkippedDuration().AsDuration(), diff --git a/chasm/lib/scheduler/migration/migration.go b/chasm/lib/scheduler/migration/migration.go index 88c63b8b453..e294978749f 100644 --- a/chasm/lib/scheduler/migration/migration.go +++ b/chasm/lib/scheduler/migration/migration.go @@ -436,12 +436,16 @@ func convertBackfillsLegacyToCHASM( for _, v1Backfill := range legacyBackfills { backfillID := schedulerinternal.GenerateBackfillerID() + // V1's StartTime is an exclusive cursor that has already been consumed, so + // carry it over as the CHASM high watermark. The watermark's presence marks + // the backfiller as having recorded progress, which stops processBackfill + // from re-including the cursor point. backfillers[backfillID] = &schedulerpb.BackfillerState{ Request: &schedulerpb.BackfillerState_BackfillRequest{ BackfillRequest: common.CloneProto(v1Backfill), }, BackfillId: backfillID, - LastProcessedTime: nil, + LastProcessedTime: common.CloneProto(v1Backfill.GetStartTime()), Attempt: 0, } } @@ -551,8 +555,13 @@ func convertBackfillersCHASMToLegacy( for _, backfiller := range backfillers { if request := backfiller.GetBackfillRequest(); request != nil { backfill := common.CloneProto(request) - if backfiller.GetAttempt() > 0 && backfiller.GetLastProcessedTime() != nil { - backfill.StartTime = common.CloneProto(backfiller.GetLastProcessedTime()) + // V1's StartTime is an exclusive cursor. Recorded progress is already + // such a cursor. A fresh backfiller's range start is inclusive, so shift + // it back to keep that point. + if lastProcessed := backfiller.GetLastProcessedTime(); lastProcessed != nil { + backfill.StartTime = common.CloneProto(lastProcessed) + } else { + backfill.StartTime = timestamppb.New(request.GetStartTime().AsTime().Add(-time.Millisecond)) } ongoing = append(ongoing, backfill) continue diff --git a/chasm/lib/scheduler/migration_backfill_test.go b/chasm/lib/scheduler/migration_backfill_test.go new file mode 100644 index 00000000000..1b69f93c9f4 --- /dev/null +++ b/chasm/lib/scheduler/migration_backfill_test.go @@ -0,0 +1,181 @@ +package scheduler_test + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + schedulepb "go.temporal.io/api/schedule/v1" + workflowpb "go.temporal.io/api/workflow/v1" + "go.temporal.io/server/api/historyservice/v1" + "go.temporal.io/server/api/historyservicemock/v1" + schedulespb "go.temporal.io/server/api/schedule/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/scheduler" + schedulerpb "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" + "go.temporal.io/server/chasm/lib/scheduler/migration" + "go.temporal.io/server/common/clock" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/testing/protorequire" + "go.temporal.io/server/common/testing/testlogger" + "go.uber.org/mock/gomock" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestMigrationBackfill_NativeResumesAfterWatermark(t *testing.T) { + checkMigrationBackfillWatermark(t, false, time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC)) +} + +func TestMigrationBackfill_MigratedResumesAfterWatermark(t *testing.T) { + for _, end := range []time.Time{time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC), time.Unix(60, 0).UTC()} { + t.Run(end.String(), func(t *testing.T) { checkMigrationBackfillWatermark(t, true, end) }) + } +} + +func checkMigrationBackfillWatermark(t *testing.T, migrate bool, when time.Time) { + t.Helper() + env := newTestEnv(t) + env.TimeSource.Update(when) + watermark := timestamppb.New(when.Add(-time.Minute)) + backfill := &schedulepb.BackfillRequest{ + StartTime: watermark, + EndTime: timestamppb.New(when), + OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL, + } + state := &schedulerpb.BackfillerState{ + Request: &schedulerpb.BackfillerState_BackfillRequest{BackfillRequest: backfill}, + LastProcessedTime: watermark, + Attempt: 1, + } + if migrate { + imported := migration.LegacyToCreateFromMigrationStateRequest( + env.Scheduler.Schedule, + &schedulepb.ScheduleInfo{ActionCount: 1}, + &schedulespb.InternalState{ + Namespace: namespace, + NamespaceId: namespaceID, + ScheduleId: scheduleID, + OngoingBackfills: []*schedulepb.BackfillRequest{backfill}, + }, + nil, + nil, + when, + ) + require.Len(t, imported.State.Backfillers, 1) + for _, converted := range imported.State.Backfillers { + state = converted + } + } + handler := scheduler.NewBackfillerTaskHandler(scheduler.BackfillerTaskHandlerOptions{ + Config: defaultConfig(), + MetricsHandler: metrics.NoopMetricsHandler, + BaseLogger: env.Logger, + SpecProcessor: env.SpecProcessor, + }) + result, err := handler.ProcessBackfill(env.Scheduler, &scheduler.Backfiller{BackfillerState: state}, 10) + require.NoError(t, err) + require.True(t, result.Complete) + require.Len(t, result.BufferedStarts, 1, "V1's incremental cursor is exclusive") + protorequire.ProtoEqual(t, timestamppb.New(when), result.BufferedStarts[0].ActualTime) +} + +func TestMigrationBackfill_WaitsForCallbackReconciliation(t *testing.T) { + boundary := time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC) + logger := testlogger.NewTestLogger(t, testlogger.FailOnExpectedErrorOnly) + timeSource := clock.NewEventTimeSource().Update(boundary.Add(time.Hour)) + engine, engineCtx := newTestEngineContext(t, logger, withEngineTimeSource(timeSource)) + schedule := defaultSchedule() + schedule.State.Paused = true + schedule.Policies.OverlapPolicy = enumspb.SCHEDULE_OVERLAP_POLICY_SKIP + request := migration.LegacyToCreateFromMigrationStateRequest( + schedule, + &schedulepb.ScheduleInfo{RunningWorkflows: []*commonpb.WorkflowExecution{{ + WorkflowId: "stale-workflow", + RunId: "stale-run", + }}}, + &schedulespb.InternalState{ + Namespace: namespace, + NamespaceId: namespaceID, + ScheduleId: scheduleID, + LastProcessedTime: timestamppb.New(timeSource.Now()), + BufferedStarts: []*schedulespb.BufferedStart{{ + NominalTime: timestamppb.New(boundary), + ActualTime: timestamppb.New(boundary), + Manual: true, + RequestId: "pending-request", + WorkflowId: "pending-workflow", + }}, + OngoingBackfills: []*schedulepb.BackfillRequest{{ + StartTime: timestamppb.New(boundary), + EndTime: timestamppb.New(boundary.Add(time.Minute)), + }}, + }, + nil, + nil, + timeSource.Now(), + ) + + _, err := scheduler.NewTestHandler(logger).TestCreateFromMigrationState(engineCtx, request) + require.NoError(t, err) + ref := chasm.NewComponentRef[*scheduler.Scheduler](chasm.ExecutionKey{ + NamespaceID: namespaceID, + BusinessID: scheduleID, + }) + _, err = chasm.ReadComponent( + engineCtx, + ref, + func(s *scheduler.Scheduler, ctx chasm.Context, _ struct{}) (struct{}, error) { + require.Len(t, s.Backfillers, 1) + require.Zero(t, s.Info.OverlapSkipped) + starts := s.Invoker.Get(ctx).BufferedStarts + require.Len(t, starts, 2) + require.Equal(t, "pending-request", starts[0].RequestId) + require.False(t, starts[1].HasCallback) + return struct{}{}, nil + }, + struct{}{}, + ) + require.NoError(t, err) + + historyClient := historyservicemock.NewMockHistoryServiceClient(gomock.NewController(t)) + historyClient.EXPECT().DescribeWorkflowExecution(gomock.Any(), gomock.Any()).Return( + &historyservice.DescribeWorkflowExecutionResponse{ + WorkflowExecutionInfo: &workflowpb.WorkflowExecutionInfo{ + Status: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, + CloseTime: timestamppb.New(boundary.Add(time.Minute)), + }, + }, + nil, + ) + callbackHandler := scheduler.NewSchedulerCallbacksTaskHandler(scheduler.SchedulerCallbacksTaskHandlerOptions{ + Config: defaultConfig(), + HistoryClient: historyClient, + MetricsHandler: metrics.NoopMetricsHandler, + }) + require.NoError(t, callbackHandler.Execute( + engineCtx, + ref, + chasm.TaskAttributes{}, + &schedulerpb.SchedulerCallbacksTask{}, + )) + _, err = engine.FirePureTasks(ref, timeSource.Now()) + require.NoError(t, err) + _, err = chasm.ReadComponent( + engineCtx, + ref, + func(s *scheduler.Scheduler, ctx chasm.Context, _ struct{}) (struct{}, error) { + require.Empty(t, s.Backfillers) + require.LessOrEqual(t, s.Info.OverlapSkipped, int64(1)) + for _, start := range s.Invoker.Get(ctx).BufferedStarts { + if start.GetRunId() == "stale-run" { + require.NotNil(t, start.Completed) + } + } + return struct{}{}, nil + }, + struct{}{}, + ) + require.NoError(t, err) +} diff --git a/chasm/lib/scheduler/migration_chain_ownership_test.go b/chasm/lib/scheduler/migration_chain_ownership_test.go new file mode 100644 index 00000000000..06c6fc595d9 --- /dev/null +++ b/chasm/lib/scheduler/migration_chain_ownership_test.go @@ -0,0 +1,58 @@ +package scheduler_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "go.temporal.io/api/serviceerror" + "go.temporal.io/server/api/historyservice/v1" + persistencespb "go.temporal.io/server/api/persistence/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/scheduler" + "go.uber.org/mock/gomock" + "google.golang.org/grpc" +) + +func TestMigrationRollbackChainOwnership(t *testing.T) { + for _, scenario := range []string{"owned_descendant", "foreign_chain", "first_run_removed"} { + t.Run(scenario, func(t *testing.T) { + e, handler, history := newRollbackScenario(t) + history.EXPECT().StartWorkflowExecution(gomock.Any(), gomock.Any()).Return(nil, + serviceerror.NewWorkflowExecutionAlreadyStarted("continued-as-new destination", "descendant-request", "descendant-run")) + history.EXPECT().GetMutableState(gomock.Any(), gomock.Any()).Return( + &historyservice.GetMutableStateResponse{FirstExecutionRunId: "first-run"}, nil).AnyTimes() + verified := false + history.EXPECT().DescribeMutableState(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, req *historyservice.DescribeMutableStateRequest, _ ...grpc.CallOption) (*historyservice.DescribeMutableStateResponse, error) { + require.Equal(t, "first-run", req.GetExecution().GetRunId()) + verified = true + if scenario == "first_run_removed" { + return nil, serviceerror.NewNotFound("original run expired") + } + owner := "rollback" + if scenario == "foreign_chain" { + owner = "foreign" + } + return &historyservice.DescribeMutableStateResponse{DatabaseMutableState: &persistencespb.WorkflowMutableState{ + ExecutionState: &persistencespb.WorkflowExecutionState{RunId: "first-run", CreateRequestId: owner}, + }}, nil + }).AnyTimes() + _, err := executeRollbackTask(t, e, handler, e.engine) + switch scenario { + case "owned_descendant": + require.NoError(t, err) + case "foreign_chain": + require.ErrorAs(t, err, new(*serviceerror.AlreadyExists)) + default: + require.ErrorAs(t, err, new(*serviceerror.FailedPrecondition)) + require.ErrorContains(t, err, "is gone") + } + require.True(t, verified, "a descendant's start request does not establish ownership of the original migration") + require.NoError(t, e.readScheduler(func(s *scheduler.Scheduler, _ chasm.Context) error { + require.Equal(t, scenario == "owned_descendant", s.Closed) + return nil + })) + }) + } +} diff --git a/chasm/lib/scheduler/migration_handoff_test.go b/chasm/lib/scheduler/migration_handoff_test.go new file mode 100644 index 00000000000..b33312c4ae9 --- /dev/null +++ b/chasm/lib/scheduler/migration_handoff_test.go @@ -0,0 +1,247 @@ +package scheduler_test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + schedulepb "go.temporal.io/api/schedule/v1" + "go.temporal.io/api/serviceerror" + "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/server/api/historyservice/v1" + "go.temporal.io/server/api/historyservicemock/v1" + persistencespb "go.temporal.io/server/api/persistence/v1" + schedulespb "go.temporal.io/server/api/schedule/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/scheduler" + schedulerpb "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" + "go.temporal.io/server/common/clock" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/payload" + "go.temporal.io/server/common/sdk" + "go.temporal.io/server/common/searchattribute" + "go.temporal.io/server/common/testing/protorequire" + "go.uber.org/mock/gomock" + "google.golang.org/grpc" +) + +type migrationCloseFaultEngine struct { + chasm.Engine + afterCommit bool + armed bool +} + +func (e *migrationCloseFaultEngine) UpdateComponent( + ctx context.Context, ref chasm.ComponentRef, update func(chasm.MutableContext, chasm.Component) error, opts ...chasm.TransitionOption, +) ([]byte, error) { + if !e.armed { + return e.Engine.UpdateComponent(ctx, ref, update, opts...) + } + e.armed = false + if e.afterCommit { + if _, err := e.Engine.UpdateComponent(ctx, ref, update, opts...); err != nil { + return nil, err + } + } + return nil, serviceerror.NewUnavailable("injected source-close response loss") +} + +func newRollbackScenario(t *testing.T) (*schedulerTestEngine, *scheduler.SchedulerMigrateToWorkflowTaskHandler, *historyservicemock.MockHistoryServiceClient) { + t.Helper() + ts := clock.NewEventTimeSource().Update(time.Date(2026, 8, 1, 12, 0, 0, 0, time.UTC)) + sched := defaultSchedule() + sched.State.Paused = true + sched.State.Notes = "user maintenance" + e := newSchedulerTestEngine(t, sched, withEngineTimeSource(ts)) + require.NoError(t, e.updateScheduler(func(s *scheduler.Scheduler, ctx chasm.MutableContext) error { + _, err := s.Patch(ctx, &schedulerpb.PatchScheduleRequest{FrontendRequest: &workflowservice.PatchScheduleRequest{ + RequestId: "trigger-request", Patch: &schedulepb.SchedulePatch{ + TriggerImmediately: &schedulepb.TriggerImmediatelyRequest{OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL}, + }, + }}) + if err != nil { + return err + } + s.Visibility.Get(ctx).MergeCustomMemo(ctx, map[string]*commonpb.Payload{"custom": payload.EncodeString("memo")}) + if err := s.Visibility.Get(ctx).MergeCustomSearchAttributes(ctx, map[string]*commonpb.Payload{"CustomKeywordField": payload.EncodeString("attribute")}); err != nil { + return err + } + _, err = s.MigrateToWorkflow(ctx, &schedulerpb.MigrateToWorkflowRequest{RequestId: "rollback"}) + return err + })) + historyClient := historyservicemock.NewMockHistoryServiceClient(gomock.NewController(t)) + handler := scheduler.NewSchedulerMigrateToWorkflowTaskHandler(scheduler.SchedulerMigrateToWorkflowTaskHandlerOptions{ + Config: defaultConfig(), MetricsHandler: metrics.NoopMetricsHandler, BaseLogger: e.logger, + HistoryClient: historyClient, SaMapperProvider: searchattribute.NewTestMapperProvider(nil), + }) + return e, handler, historyClient +} + +func executeRollbackTask(t *testing.T, e *schedulerTestEngine, handler *scheduler.SchedulerMigrateToWorkflowTaskHandler, engine chasm.Engine) (bool, error) { + t.Helper() + var valid bool + err := e.readScheduler(func(s *scheduler.Scheduler, ctx chasm.Context) error { + var err error + valid, err = handler.Validate(ctx, s, chasm.TaskInvocation{}, &schedulerpb.SchedulerMigrateToWorkflowTask{}) + return err + }) + if err != nil || !valid { + return !valid, err + } + return false, handler.Execute(chasm.NewEngineContext(context.Background(), engine), e.rootRef, chasm.TaskAttributes{}, &schedulerpb.SchedulerMigrateToWorkflowTask{}) +} + +func TestMigrationScenario_RollbackRetryBoundaries(t *testing.T) { + for _, boundary := range []string{"success", "before_destination", "after_destination", "before_source_close", "after_source_close"} { + t.Run(boundary, func(t *testing.T) { + e, handler, historyClient := newRollbackScenario(t) + faultEngine := &migrationCloseFaultEngine{Engine: e.engine, + armed: boundary == "before_source_close" || boundary == "after_source_close", afterCommit: boundary == "after_source_close"} + var destination *schedulespb.StartScheduleArgs + var firstRequestID string + attempts, creations := 0, 0 + historyClient.EXPECT().StartWorkflowExecution(gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn( + func(_ context.Context, req *historyservice.StartWorkflowExecutionRequest, _ ...grpc.CallOption) (*historyservice.StartWorkflowExecutionResponse, error) { + attempts++ + if attempts == 1 { + firstRequestID = req.StartRequest.RequestId + } + if attempts == 1 && boundary == "before_destination" { + return nil, serviceerror.NewUnavailable("injected before destination commit") + } + if destination != nil { + return nil, serviceerror.NewWorkflowExecutionAlreadyStarted("retry", firstRequestID, "destination-run") + } + destination = &schedulespb.StartScheduleArgs{} + require.NoError(t, sdk.PreferProtoDataConverter.FromPayloads(req.StartRequest.Input, destination)) + protorequire.ProtoEqual(t, payload.EncodeString("memo"), req.StartRequest.Memo.Fields["custom"]) + protorequire.ProtoEqual(t, payload.EncodeString("attribute"), req.StartRequest.SearchAttributes.IndexedFields["CustomKeywordField"]) + creations++ + if attempts == 1 && boundary == "after_destination" { + return nil, serviceerror.NewUnavailable("injected after destination commit") + } + return &historyservice.StartWorkflowExecutionResponse{RunId: "destination-run"}, nil + }) + dropped, err := executeRollbackTask(t, e, handler, faultEngine) + require.False(t, dropped) + if boundary == "success" { + require.NoError(t, err) + } else { + require.Error(t, err) + _, err = executeRollbackTask(t, e, handler, faultEngine) + require.NoError(t, err) + } + require.Equal(t, 1, creations) + require.NotNil(t, destination) + require.True(t, destination.Schedule.State.Paused) + require.Equal(t, "user maintenance", destination.Schedule.State.Notes) + require.Len(t, destination.State.BufferedStarts, 1) + require.True(t, destination.State.BufferedStarts[0].Manual) + require.NoError(t, e.readScheduler(func(s *scheduler.Scheduler, _ chasm.Context) error { + require.True(t, s.Closed) + require.Nil(t, s.WorkflowMigration) + return nil + })) + dropped, err = executeRollbackTask(t, e, handler, faultEngine) + require.NoError(t, err) + require.True(t, dropped, "duplicate task must not create another destination") + }) + } +} + +func TestMigrationRollbackDestinationCollision(t *testing.T) { + e, handler, historyClient := newRollbackScenario(t) + historyClient.EXPECT().StartWorkflowExecution(gomock.Any(), gomock.Any()).Return(nil, + serviceerror.NewWorkflowExecutionAlreadyStarted("unrelated scheduler owns this ID", "unrelated-create-request", "unrelated-run")) + historyClient.EXPECT().GetMutableState(gomock.Any(), gomock.Any()).Return( + &historyservice.GetMutableStateResponse{FirstExecutionRunId: "unrelated-run"}, nil) + historyClient.EXPECT().DescribeMutableState(gomock.Any(), gomock.Any()).Return( + &historyservice.DescribeMutableStateResponse{DatabaseMutableState: &persistencespb.WorkflowMutableState{ + ExecutionState: &persistencespb.WorkflowExecutionState{RunId: "unrelated-run", CreateRequestId: "unrelated-create-request"}, + }}, nil) + _, err := executeRollbackTask(t, e, handler, e.engine) + require.NoError(t, e.readScheduler(func(s *scheduler.Scheduler, _ chasm.Context) error { + require.False(t, s.Closed, "rollback must retain the source and its acknowledged trigger when a different workflow owns the destination ID") + return nil + })) + require.ErrorAs(t, err, new(*serviceerror.AlreadyExists), "an unrelated destination is a conflict, not a successful handoff") +} + +func TestMigrationRollbackSendsStoredRequestID(t *testing.T) { + e, handler, historyClient := newRollbackScenario(t) + var requestIDs []string + historyClient.EXPECT().StartWorkflowExecution(gomock.Any(), gomock.Any()).Times(2).DoAndReturn( + func(_ context.Context, req *historyservice.StartWorkflowExecutionRequest, _ ...grpc.CallOption) (*historyservice.StartWorkflowExecutionResponse, error) { + requestIDs = append(requestIDs, req.StartRequest.RequestId) + if len(requestIDs) == 1 { + return nil, serviceerror.NewUnavailable("injected") + } + return &historyservice.StartWorkflowExecutionResponse{RunId: "destination-run"}, nil + }) + _, err := executeRollbackTask(t, e, handler, e.engine) + require.Error(t, err) + _, err = executeRollbackTask(t, e, handler, e.engine) + require.NoError(t, err) + require.Equal(t, []string{"rollback", "rollback"}, requestIDs) +} + +func TestMigrationRollbackAssignsRequestIDToLegacyMigration(t *testing.T) { + e, handler, historyClient := newRollbackScenario(t) + require.NoError(t, e.updateScheduler(func(s *scheduler.Scheduler, _ chasm.MutableContext) error { + s.WorkflowMigration.RequestId = "" + return nil + })) + var requestID string + historyClient.EXPECT().StartWorkflowExecution(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, req *historyservice.StartWorkflowExecutionRequest, _ ...grpc.CallOption) (*historyservice.StartWorkflowExecutionResponse, error) { + requestID = req.StartRequest.RequestId + return nil, serviceerror.NewUnavailable("injected") + }) + _, err := executeRollbackTask(t, e, handler, e.engine) + require.Error(t, err) + require.NotEmpty(t, requestID) + require.NoError(t, e.readScheduler(func(s *scheduler.Scheduler, _ chasm.Context) error { + require.Equal(t, requestID, s.WorkflowMigration.GetRequestId(), "the ID must be persisted before the first start attempt") + require.False(t, s.Closed) + return nil + })) +} + +func TestMigrationRollbackLegacyMigrationRejectsUnprovenDestination(t *testing.T) { + e, handler, historyClient := newRollbackScenario(t) + require.NoError(t, e.updateScheduler(func(s *scheduler.Scheduler, _ chasm.MutableContext) error { + s.WorkflowMigration.RequestId = "" + return nil + })) + // An attempt made before the upgrade may have started the workflow under a random ID. + historyClient.EXPECT().StartWorkflowExecution(gomock.Any(), gomock.Any()).Return(nil, + serviceerror.NewWorkflowExecutionAlreadyStarted("old attempt", "random-pre-upgrade-id", "old-run")) + historyClient.EXPECT().GetMutableState(gomock.Any(), gomock.Any()).Return( + &historyservice.GetMutableStateResponse{FirstExecutionRunId: "old-run"}, nil) + historyClient.EXPECT().DescribeMutableState(gomock.Any(), gomock.Any()).Return( + &historyservice.DescribeMutableStateResponse{DatabaseMutableState: &persistencespb.WorkflowMutableState{ + ExecutionState: &persistencespb.WorkflowExecutionState{RunId: "old-run", CreateRequestId: "random-pre-upgrade-id"}, + }}, nil) + _, err := executeRollbackTask(t, e, handler, e.engine) + require.ErrorAs(t, err, new(*serviceerror.AlreadyExists)) + require.NoError(t, e.readScheduler(func(s *scheduler.Scheduler, _ chasm.Context) error { + require.False(t, s.Closed) + return nil + })) +} + +func TestMigrateToWorkflowGeneratesRequestID(t *testing.T) { + s, ctx, _ := setupSchedulerForTest(t) + _, err := s.MigrateToWorkflow(ctx, &schedulerpb.MigrateToWorkflowRequest{}) + require.NoError(t, err) + require.NotEmpty(t, s.WorkflowMigration.GetRequestId()) + stored := s.WorkflowMigration.GetRequestId() + + // A repeated request does not replace the stored ID. + _, err = s.MigrateToWorkflow(ctx, &schedulerpb.MigrateToWorkflowRequest{RequestId: "other"}) + require.NoError(t, err) + require.Equal(t, stored, s.WorkflowMigration.GetRequestId()) +} diff --git a/chasm/lib/scheduler/migration_rollback_phase_test.go b/chasm/lib/scheduler/migration_rollback_phase_test.go new file mode 100644 index 00000000000..ba15644c8ab --- /dev/null +++ b/chasm/lib/scheduler/migration_rollback_phase_test.go @@ -0,0 +1,128 @@ +package scheduler_test + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + schedulespb "go.temporal.io/server/api/schedule/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/scheduler" + schedulerpb "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" + "go.temporal.io/server/chasm/lib/scheduler/migration" + "go.temporal.io/server/common" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/testing/protorequire" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestRollbackPhaseNativeControl(t *testing.T) { + for _, phase := range []string{"ready", "retry", "cancel", "terminate", "mixed"} { + t.Run(phase, func(t *testing.T) { + s, ctx, _ := setupSchedulerForTest(t) + i := s.Invoker.Get(ctx) + i.InvokerState = rollbackPhaseState(phase) + h := scheduler.NewInvokerExecuteTaskHandler(scheduler.InvokerTaskHandlerOptions{Config: defaultConfig()}) + if phase == "retry" { + i.LastProcessedTime = i.BufferedStarts[0].BackoffTime + } + valid, err := h.Validate(ctx, i, chasm.TaskInvocation{}, &schedulerpb.InvokerExecuteTask{}) + require.NoError(t, err) + require.True(t, valid) + }) + } +} + +func TestRollbackPhaseGate(t *testing.T) { + for _, phase := range []string{"ready", "retry", "cancel", "terminate", "mixed"} { + t.Run(phase, func(t *testing.T) { + s, ctx, _ := setupSchedulerForTest(t) + i := s.Invoker.Get(ctx) + i.InvokerState = rollbackPhaseState(phase) + before := common.CloneProto(i.InvokerState) + _, err := s.MigrateToWorkflow(ctx, &schedulerpb.MigrateToWorkflowRequest{}) + require.ErrorIs(t, err, scheduler.ErrMigrationActionsPending, "case=%s: V1 cannot execute persisted invoker phase; retain native task ownership", phase) + require.Nil(t, s.WorkflowMigration) + require.False(t, s.Schedule.State.Paused) + protorequire.ProtoEqual(t, before, i.InvokerState) + }) + } +} + +func rollbackPhaseState(phase string) *schedulerpb.InvokerState { + now := timestamppb.New(time.Date(2026, 9, 4, 12, 0, 0, 0, time.UTC)) + i := &schedulerpb.InvokerState{LastProcessedTime: now} + if phase == "ready" || phase == "retry" || phase == "mixed" { + i.BufferedStarts = []*schedulespb.BufferedStart{{RequestId: "pending-request", WorkflowId: "pending-workflow", NominalTime: now, ActualTime: now, DesiredTime: now, Manual: true, Attempt: 1, OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL}} + } + if phase == "retry" { + i.BufferedStarts[0].Attempt = 3 + i.BufferedStarts[0].BackoffTime = timestamppb.New(now.AsTime().Add(time.Hour)) + } + if phase == "cancel" || phase == "mixed" { + i.CancelWorkflows = []*commonpb.WorkflowExecution{{WorkflowId: "cancel-workflow", RunId: "cancel-run"}} + } + if phase == "terminate" || phase == "mixed" { + i.TerminateWorkflows = []*commonpb.WorkflowExecution{{WorkflowId: "terminate-workflow", RunId: "terminate-run"}} + } + return i +} + +func TestRollbackPhaseConversionPreservesFieldsButLosesQueues(t *testing.T) { + s, ctx, _ := setupSchedulerForTest(t) + i := rollbackPhaseState("mixed") + args := migration.CHASMToLegacyStartScheduleArgs(s.SchedulerState, s.Generator.Get(ctx).GeneratorState, i, nil, nil, nil, nil, ctx.Now(s)) + // InternalState has no cancel/terminate queues, so only buffered starts carry over. + require.Len(t, args.State.BufferedStarts, 1) + protorequire.ProtoEqual(t, i.BufferedStarts[0], args.State.BufferedStarts[0]) + require.Empty(t, args.Info.RunningWorkflows) +} + +func TestRollbackPhaseTransferable(t *testing.T) { + for _, phase := range []string{"unprocessed", "deferred", "running", "completed", "drained"} { + t.Run(phase, func(t *testing.T) { + s, ctx, _ := setupSchedulerForTest(t) + i := s.Invoker.Get(ctx) + i.InvokerState = rollbackPhaseState("ready") + start := i.BufferedStarts[0] + switch phase { + case "unprocessed": + start.Attempt = 0 + case "deferred": + start.Attempt = -1 + case "running": + start.RunId = "run" + case "completed": + start.RunId = "run" + start.Completed = &schedulespb.CompletedResult{} + case "drained": + i.BufferedStarts = nil + default: + t.Fatalf("unknown phase %q", phase) + } + _, err := s.MigrateToWorkflow(ctx, &schedulerpb.MigrateToWorkflowRequest{}) + require.NoError(t, err) + require.NotNil(t, s.WorkflowMigration) + }) + } +} + +func TestRollbackPhaseMigrateTaskRefusesPendingActions(t *testing.T) { + e := newSchedulerTestEngine(t, defaultSchedule()) + require.NoError(t, e.updateScheduler(func(s *scheduler.Scheduler, ctx chasm.MutableContext) error { + s.Invoker.Get(ctx).InvokerState = rollbackPhaseState("mixed") + s.WorkflowMigration = &schedulerpb.WorkflowMigrationState{} + s.Schedule.State.Paused = true + return nil + })) + h := scheduler.NewSchedulerMigrateToWorkflowTaskHandler(scheduler.SchedulerMigrateToWorkflowTaskHandlerOptions{Config: defaultConfig(), MetricsHandler: metrics.NoopMetricsHandler, BaseLogger: e.logger}) + err := h.Execute(e.engineCtx, e.rootRef, chasm.TaskAttributes{}, &schedulerpb.SchedulerMigrateToWorkflowTask{}) + require.ErrorIs(t, err, scheduler.ErrMigrationActionsPending) + require.NoError(t, e.readScheduler(func(s *scheduler.Scheduler, _ chasm.Context) error { + require.False(t, s.Closed) + require.NotNil(t, s.WorkflowMigration) + return nil + })) +} diff --git a/chasm/lib/scheduler/proto/v1/message.proto b/chasm/lib/scheduler/proto/v1/message.proto index 3bfbbaadc03..39c1fca8ef0 100644 --- a/chasm/lib/scheduler/proto/v1/message.proto +++ b/chasm/lib/scheduler/proto/v1/message.proto @@ -68,6 +68,12 @@ message WorkflowMigrationState { // The schedule's notes before migration was initiated. string pre_migration_notes = 2; + + // Request ID used for every StartWorkflowExecution attempt of the destination + // V1 workflow. It identifies the destination as created by this migration. + // Empty only on migrations persisted before this field existed; the migrate + // task assigns one before its first start attempt. + string request_id = 3; } // CHASM scheduler's Generator internal state. diff --git a/chasm/lib/scheduler/result_reordering_test.go b/chasm/lib/scheduler/result_reordering_test.go new file mode 100644 index 00000000000..bb90a03b6e2 --- /dev/null +++ b/chasm/lib/scheduler/result_reordering_test.go @@ -0,0 +1,33 @@ +package scheduler_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + enumspb "go.temporal.io/api/enums/v1" + schedulespb "go.temporal.io/server/api/schedule/v1" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestExecuteResultsFollowIdentityAcrossBufferMovement(t *testing.T) { + t.Parallel() + env := newTestEnv(t) + ctx := env.MutableContext() + invoker := env.Scheduler.Invoker.Get(ctx) + now := timestamppb.New(env.TimeSource.Now()) + first := &schedulespb.BufferedStart{RequestId: "first", WorkflowId: "first-workflow", Attempt: 1, OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ALL} + second := &schedulespb.BufferedStart{RequestId: "second", WorkflowId: "second-workflow", Attempt: 1, OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ALL} + invoker.BufferedStarts = []*schedulespb.BufferedStart{second, first} + result := []*schedulespb.BufferedStart{ + {RequestId: "first", RunId: "first-run", StartTime: now}, + {RequestId: "second", RunId: "second-run", StartTime: now}, + } + started, duplicates, _ := invoker.RecordExecuteResult(ctx, result, nil) + require.Equal(t, 2, started) + require.Zero(t, duplicates) + require.Equal(t, "second-run", invoker.BufferedStarts[0].GetRunId()) + require.Equal(t, "first-run", invoker.BufferedStarts[1].GetRunId()) + started, duplicates, _ = invoker.RecordExecuteResult(ctx, result, nil) + require.Zero(t, started) + require.Equal(t, 2, duplicates) +} diff --git a/chasm/lib/scheduler/scheduler.go b/chasm/lib/scheduler/scheduler.go index e97083d2151..7383ce637a4 100644 --- a/chasm/lib/scheduler/scheduler.go +++ b/chasm/lib/scheduler/scheduler.go @@ -7,6 +7,7 @@ import ( "slices" "time" + "github.com/google/uuid" commonpb "go.temporal.io/api/common/v1" enumspb "go.temporal.io/api/enums/v1" schedulepb "go.temporal.io/api/schedule/v1" @@ -115,7 +116,9 @@ var ( ErrSentinel = serviceerror.NewNotFound("schedule is a sentinel") ErrSentinelBlocked = serviceerror.NewUnavailable("schedule is a sentinel; please retry after sentinel expires") ErrMigrationPending = serviceerror.NewUnavailable("schedule has a pending migration to workflow; please retry later") - ErrTimeSkippingMigration = serviceerror.NewFailedPrecondition("schedule with time skipping cannot migrate to workflow-backed scheduler") + // ErrMigrationActionsPending is retryable: the invoker finishes the actions and a later attempt succeeds. + ErrMigrationActionsPending = serviceerror.NewUnavailable("schedule has in-flight actions; retry migration after they finish") + ErrTimeSkippingMigration = serviceerror.NewFailedPrecondition("schedule with time skipping cannot migrate to workflow-backed scheduler") ) // NewScheduler returns an initialized CHASM scheduler root component. @@ -325,8 +328,17 @@ func CreateSchedulerFromMigration( sched.Invoker = chasm.NewComponentField(ctx, newInvokerWithState(ctx, state.GetInvokerState())) sched.Generator = chasm.NewComponentField(ctx, newGeneratorWithState(ctx, state.GetGeneratorState())) + // Backfillers also wait for SchedulerCallbacksTask when it is needed, so they + // don't buffer starts against stale running-workflow state. + needsCallbacks := slices.ContainsFunc(state.GetInvokerState().GetBufferedStarts(), needsCallback) for backfillID, backfillerState := range state.GetBackfillers() { - sched.Backfillers[backfillID] = chasm.NewComponentField(ctx, newBackfillerWithState(ctx, backfillerState)) + var backfiller *Backfiller + if needsCallbacks { + backfiller = newBackfillerWithStateWithoutTask(ctx, backfillerState) + } else { + backfiller = newBackfillerWithState(ctx, backfillerState) + } + sched.Backfillers[backfillID] = chasm.NewComponentField(ctx, backfiller) } visibility, err := chasm.NewVisibilityWithData(ctx, state.GetSearchAttributes(), state.GetMemo()) @@ -337,7 +349,7 @@ func CreateSchedulerFromMigration( // Defer generation until SchedulerCallbacksTask resolves stale running-workflow // state; if there are none, generate directly. - if slices.ContainsFunc(state.GetInvokerState().GetBufferedStarts(), needsCallback) { + if needsCallbacks { ctx.AddTask(sched, chasm.TaskAttributes{}, &schedulerpb.SchedulerCallbacksTask{}) } else { sched.Generator.Get(ctx).Generate(ctx) @@ -879,10 +891,22 @@ func (s *Scheduler) MigrateToWorkflow( return nil, ErrTimeSkippingMigration } + if !s.Invoker.Get(ctx).canMigrateToWorkflow() { + return nil, ErrMigrationActionsPending + } + + // The request ID is sent on every start attempt of the destination workflow, + // so a retry can recognize its own workflow. + requestID := req.GetRequestId() + if requestID == "" { + requestID = uuid.NewString() + } + // Save pre-migration paused state, mark migration as pending, then pause. s.WorkflowMigration = &schedulerpb.WorkflowMigrationState{ PreMigrationPaused: s.Schedule.State.Paused, PreMigrationNotes: s.Schedule.State.Notes, + RequestId: requestID, } s.Schedule.State.Paused = true s.Schedule.State.Notes = "paused for migration to workflow-backed scheduler" diff --git a/chasm/lib/scheduler/scheduler_migrate_task.go b/chasm/lib/scheduler/scheduler_migrate_task.go index 536b41b4fc8..748eceae923 100644 --- a/chasm/lib/scheduler/scheduler_migrate_task.go +++ b/chasm/lib/scheduler/scheduler_migrate_task.go @@ -12,6 +12,7 @@ import ( "go.temporal.io/api/serviceerror" taskqueuepb "go.temporal.io/api/taskqueue/v1" "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/server/api/historyservice/v1" schedulespb "go.temporal.io/server/api/schedule/v1" "go.temporal.io/server/chasm" schedulerpb "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" @@ -112,6 +113,7 @@ func (h *SchedulerMigrateToWorkflowTaskHandler) Execute( searchAttributes map[string]*commonpb.Payload memo map[string]*commonpb.Payload now time.Time + requestID string } var result readResult @@ -119,6 +121,10 @@ func (h *SchedulerMigrateToWorkflowTaskHandler) Execute( ctx, schedulerRef, func(s *Scheduler, ctx chasm.Context, _ any) (struct{}, error) { + // The task retries until the invoker has drained. + if !s.Invoker.Get(ctx).canMigrateToWorkflow() { + return struct{}{}, ErrMigrationActionsPending + } now := ctx.Now(s) schedulerState := common.CloneProto(s.SchedulerState) generatorState := common.CloneProto(s.Generator.Get(ctx).GeneratorState) @@ -159,6 +165,7 @@ func (h *SchedulerMigrateToWorkflowTaskHandler) Execute( searchAttributes: searchAttributes, memo: memo, now: now, + requestID: schedulerState.GetWorkflowMigration().GetRequestId(), } return struct{}{}, nil }, @@ -168,6 +175,17 @@ func (h *SchedulerMigrateToWorkflowTaskHandler) Execute( return fmt.Errorf("failed to read scheduler state: %w", err) } + if result.requestID == "" { + // The migration was persisted before request IDs existed, so an earlier + // attempt may have started a workflow under a random ID. Assign an ID + // now; if such a workflow exists, the ownership check below rejects it + // rather than accepting it unproven. + result.requestID, err = h.assignRequestID(ctx, schedulerRef) + if err != nil { + return err + } + } + logger = log.With( h.baseLogger, tag.WorkflowNamespace(result.namespace), @@ -205,7 +223,7 @@ func (h *SchedulerMigrateToWorkflowTaskHandler) Execute( } workflowID := legacyscheduler.WorkflowIDPrefix + result.scheduleID startReq := &workflowservice.StartWorkflowExecutionRequest{ - RequestId: uuid.NewString(), + RequestId: result.requestID, Namespace: result.namespace, WorkflowId: workflowID, WorkflowType: &commonpb.WorkflowType{Name: legacyscheduler.WorkflowType}, @@ -224,10 +242,17 @@ func (h *SchedulerMigrateToWorkflowTaskHandler) Execute( common.CreateHistoryStartWorkflowRequest(result.namespaceID, startReq, nil, nil, result.now), ) if err != nil { - // Treat already-started as success for idempotency. - if _, ok := errors.AsType[*serviceerror.WorkflowExecutionAlreadyStarted](err); !ok { + alreadyStarted, ok := errors.AsType[*serviceerror.WorkflowExecutionAlreadyStarted](err) + if !ok { return fmt.Errorf("failed to start V1 scheduler workflow: %w", err) } + // Only a workflow created by this migration is an idempotent success. + if alreadyStarted.StartRequestId != result.requestID { + err = h.verifyDestinationOwner(ctx, result.namespaceID, workflowID, alreadyStarted.RunId, result.requestID) + if err != nil { + return err + } + } } // Mark the CHASM scheduler as closed now that the V1 workflow is running. @@ -247,3 +272,67 @@ func (h *SchedulerMigrateToWorkflowTaskHandler) Execute( return nil } + +// assignRequestID persists a request ID for a migration that has none and +// returns the stored ID. It keeps an ID that a concurrent attempt already stored. +func (h *SchedulerMigrateToWorkflowTaskHandler) assignRequestID( + ctx context.Context, + schedulerRef chasm.ComponentRef, +) (string, error) { + requestID, _, err := chasm.UpdateComponent( + ctx, + schedulerRef, + func(s *Scheduler, _ chasm.MutableContext, _ any) (string, error) { + if s.Closed || s.WorkflowMigration == nil { + return "", ErrMigrationPending + } + if s.WorkflowMigration.RequestId == "" { + s.WorkflowMigration.RequestId = uuid.NewString() + } + return s.WorkflowMigration.RequestId, nil + }, + nil, + ) + if err != nil { + return "", fmt.Errorf("failed to assign migration request ID: %w", err) + } + return requestID, nil +} + +// verifyDestinationOwner checks that the V1 workflow chain containing runID was +// created by this migration. The current run's start request ID differs from the +// migration's after continue-as-new, so the check compares the chain's first +// run against the migration's request ID. It returns an error unless that +// ownership is proven. +func (h *SchedulerMigrateToWorkflowTaskHandler) verifyDestinationOwner( + ctx context.Context, + namespaceID, workflowID, runID, requestID string, +) error { + current, err := h.historyClient.GetMutableState(ctx, &historyservice.GetMutableStateRequest{ + NamespaceId: namespaceID, + Execution: &commonpb.WorkflowExecution{WorkflowId: workflowID, RunId: runID}, + }) + if err != nil { + return fmt.Errorf("failed to resolve V1 scheduler workflow chain: %w", err) + } + firstRunID := current.GetFirstExecutionRunId() + if firstRunID == "" { + return serviceerror.NewFailedPreconditionf("first run of V1 scheduler workflow %q is unknown", workflowID) + } + first, err := h.historyClient.DescribeMutableState(ctx, &historyservice.DescribeMutableStateRequest{ + NamespaceId: namespaceID, + Execution: &commonpb.WorkflowExecution{WorkflowId: workflowID, RunId: firstRunID}, + }) + if _, ok := errors.AsType[*serviceerror.NotFound](err); ok { + // Without the first run, ownership cannot be proven. + return serviceerror.NewFailedPreconditionf("first run %q of V1 scheduler workflow %q is gone; cannot verify it was created by this migration", firstRunID, workflowID) + } + if err != nil { + return fmt.Errorf("failed to verify V1 scheduler workflow owner: %w", err) + } + state := first.GetDatabaseMutableState().GetExecutionState() + if state.GetRunId() != firstRunID || state.GetCreateRequestId() != requestID { + return serviceerror.NewAlreadyExistsf("V1 scheduler workflow %q was not created by this migration", workflowID) + } + return nil +} diff --git a/chasm/lib/scheduler/scheduler_migrate_test.go b/chasm/lib/scheduler/scheduler_migrate_test.go index 109c40322b1..4d4187b1f5a 100644 --- a/chasm/lib/scheduler/scheduler_migrate_test.go +++ b/chasm/lib/scheduler/scheduler_migrate_test.go @@ -100,7 +100,7 @@ func TestMigrateToWorkflowTask_DoesNotPropagateDisabledTimeSkipping(t *testing.T }, } } - env.Scheduler.WorkflowMigration = &schedulerpb.WorkflowMigrationState{} + env.Scheduler.WorkflowMigration = &schedulerpb.WorkflowMigrationState{RequestId: "migration-request"} readCtx := env.ReadContext() env.ExpectReadComponent(readCtx, env.Scheduler) diff --git a/chasm/lib/scheduler/scheduler_stuck_regression_test.go b/chasm/lib/scheduler/scheduler_stuck_regression_test.go index c56dfa83ba8..36cf57493dd 100644 --- a/chasm/lib/scheduler/scheduler_stuck_regression_test.go +++ b/chasm/lib/scheduler/scheduler_stuck_regression_test.go @@ -9,6 +9,7 @@ import ( "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/lib/scheduler" "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" + "go.temporal.io/server/chasm/lib/scheduler/schedulertest" "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -46,7 +47,7 @@ func TestIdleDeadlineRearmedAfterRecordingStart(t *testing.T) { armedDeadline := sched.GetIdleCloseTime() require.NotNil(t, armedDeadline, "precondition: exhausted schedule should arm an idle deadline") - require.NotZero(t, idleTaskCount(env.Registry, env.Node), "precondition: an idle task should be armed") + require.NotZero(t, schedulertest.IdleTaskCount(env.Registry, env.Node), "precondition: an idle task should be armed") // Record the start through the real invoker path. ctx2 := env.MutableContext() @@ -80,7 +81,7 @@ func TestIdleDeadlineRearmedAfterRecordingStart(t *testing.T) { // The schedule must still have a path to closing: a replacement idle task, // armed from the new deadline. - require.NotZero(t, idleTaskCount(env.Registry, env.Node), + require.NotZero(t, schedulertest.IdleTaskCount(env.Registry, env.Node), "recording a start invalidated the idle task with no replacement armed, while "+ "IdleCloseTime still advertises %v: the schedule will never close and the "+ "stuck-schedule scanner is told otherwise", sched.GetIdleCloseTime().AsTime()) diff --git a/chasm/lib/scheduler/scheduler_tasks.go b/chasm/lib/scheduler/scheduler_tasks.go index 4d0d5506f34..604839fa74d 100644 --- a/chasm/lib/scheduler/scheduler_tasks.go +++ b/chasm/lib/scheduler/scheduler_tasks.go @@ -237,6 +237,9 @@ func (r *SchedulerCallbacksTaskHandler) Execute( // Now that running workflow state has been refreshed, scheduler tasks can be // fired. + for _, backfillerField := range s.Backfillers { + backfillerField.Get(ctx).scheduleInitialTask(ctx) + } invoker.addTasks(ctx) generator.Generate(ctx) diff --git a/chasm/lib/scheduler/schedulertest/action_budget_test.go b/chasm/lib/scheduler/schedulertest/action_budget_test.go new file mode 100644 index 00000000000..ecef1304dc1 --- /dev/null +++ b/chasm/lib/scheduler/schedulertest/action_budget_test.go @@ -0,0 +1,51 @@ +package schedulertest_test + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + enumspb "go.temporal.io/api/enums/v1" + schedulepb "go.temporal.io/api/schedule/v1" + "go.temporal.io/server/chasm/lib/scheduler" + "go.temporal.io/server/chasm/lib/scheduler/schedulertest" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// TestActionBudgetSharedAcrossPhases pins a bug found by +// TestSchedulerProperties. InvokerExecuteTask.Execute used to pass its +// invokerTaskHandlerContext by value to terminateWorkflows, cancelWorkflows and +// startWorkflows, so each phase spent from its own copy of the action budget. +// Here one execution has a cancel, a terminate and more ALLOW_ALL starts than +// the default MaxActionsPerExecution allows; together they must stay within it. +func TestActionBudgetSharedAcrossPhases(t *testing.T) { + start := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) + s := schedulertest.DefaultSchedule() + s.Policies.OverlapPolicy = enumspb.SCHEDULE_OVERLAP_POLICY_SKIP + s.State.Paused = true + var vs []schedulertest.Violation + d := schedulertest.NewDriver(t, + schedulertest.WithStartTime(start), + schedulertest.WithSchedule(s), + schedulertest.WithAfterStep(collectViolations(&vs)), + ) + backfill := func(policy enumspb.ScheduleOverlapPolicy, points int) { + from := start.Add(-time.Hour) + require.NoError(t, d.Backfill(&schedulepb.BackfillRequest{ + StartTime: timestamppb.New(from), + EndTime: timestamppb.New(from.Add(time.Duration(points-1) * schedulertest.DefaultInterval)), + OverlapPolicy: policy, + })) + } + + require.NoError(t, d.TriggerImmediately()) + require.True(t, d.StepSideEffect(), "start the triggered workflow") + backfill(enumspb.SCHEDULE_OVERLAP_POLICY_CANCEL_OTHER, 1) + backfill(enumspb.SCHEDULE_OVERLAP_POLICY_TERMINATE_OTHER, 1) + backfill(enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL, scheduler.DefaultTweakables.MaxActionsPerExecution) + for range 4 { + d.Step() + } + + require.Empty(t, violationNames(vs)) +} diff --git a/chasm/lib/scheduler/schedulertest/driver.go b/chasm/lib/scheduler/schedulertest/driver.go new file mode 100644 index 00000000000..e73f2738a97 --- /dev/null +++ b/chasm/lib/scheduler/schedulertest/driver.go @@ -0,0 +1,497 @@ +package schedulertest + +import ( + "errors" + "time" + + commonpb "go.temporal.io/api/common/v1" + schedulepb "go.temporal.io/api/schedule/v1" + "go.temporal.io/api/serviceerror" + "go.temporal.io/api/workflowservice/v1" + persistencespb "go.temporal.io/server/api/persistence/v1" + schedulespb "go.temporal.io/server/api/schedule/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/chasmtest" + "go.temporal.io/server/chasm/lib/scheduler" + schedulerpb "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" + "go.temporal.io/server/common" + "go.temporal.io/server/common/clock" + "go.temporal.io/server/common/metrics/metricstest" + "go.temporal.io/server/common/testing/testlogger" + "go.uber.org/mock/gomock" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// Driver steps a real CHASM scheduler through every task it schedules for +// itself, advancing a controllable clock. It is a [SchedulerEngine] plus what is +// specific to driving the scheduler's lifecycle: fake frontend and history +// services with an external ledger and fault injection, the hooks that observe +// each step, and the operations a customer or the system performs on a schedule. +// After each settled transition it invokes AfterStep (if set), which is where +// invariant assertions are plugged in. +// +// A "step", run by a [chasmtest.TaskRunner], fires every pure task due at the +// current reference time and every side-effect task whose visibility time has +// arrived, then advances the clock to the next scheduled task. A healthy interval +// schedule never truly quiesces (it always re-arms a generator task), so callers +// bound the run with maxSteps; quiescence is reached only for paused, exhausted +// or closed schedules. +type Driver struct { + *SchedulerEngine + *fakeServices + + recordCalls chan *RecordCall + recorded map[string]ExternalStart + replacingSeen map[string]struct{} + initialActionCount int64 + + // metricsCapture records the scheduler's metrics; with the start ledger it + // backs the accounting invariants. + metricsCapture *metricstest.Capture + + // runner steps the execution through its pure and side-effect tasks. + runner *chasmtest.TaskRunner + + // actionsBefore is the action RPC count when the current side-effect task + // began. maxActionsObserved is the high-water mark of action RPCs made by any + // single side-effect task dispatch (one InvokerExecuteTask), which backs the + // action-budget invariant. + actionsBefore int + maxActionsObserved int + + // maxRunDuration auto-completes a started workflow before the clock moves + // past its StartTime plus this. Zero means workflows only complete through + // CompleteWorkflow. + maxRunDuration time.Duration + + // AfterStep, if set, is called after every step's CloseTransaction with the + // driver, so invariant checks can observe each settled state. + AfterStep func(d *Driver) +} + +type driverConfig struct { + schedule *schedulepb.Schedule + startTime time.Time + afterStep func(d *Driver) + config *scheduler.Config + maxLag time.Duration + maxRun time.Duration + controlledStarts bool + noStateCheck bool +} + +// DriverOption configures a Driver at construction. +type DriverOption func(*driverConfig) + +// WithSchedule overrides the schedule definition (default: DefaultSchedule). +func WithSchedule(s *schedulepb.Schedule) DriverOption { + return func(c *driverConfig) { c.schedule = s } +} + +// WithStartTime sets the wall-clock instant the engine's clock starts at. +func WithStartTime(t time.Time) DriverOption { + return func(c *driverConfig) { c.startTime = t } +} + +// WithAfterStep installs a hook called after every step. +func WithAfterStep(fn func(d *Driver)) DriverOption { + return func(c *driverConfig) { c.afterStep = fn } +} + +// WithConfig overrides the scheduler config (default: DefaultConfig). Use it to +// set small Tweakables — e.g. MaxBufferSize or MaxActionsPerExecution — so the +// buffer-bound and action-budget invariants are meaningfully exercised. +func WithConfig(cfg *scheduler.Config) DriverOption { + return func(c *driverConfig) { c.config = cfg } +} + +// WithoutStateInvariant leaves the shared stuck and idle-deadline invariants +// off the engine, which otherwise fails the test the moment a transition commits +// a violating state. Use it only in a test that corrupts the scheduler on +// purpose, and assert Driver.StateViolations instead. +func WithoutStateInvariant() DriverOption { + return func(c *driverConfig) { c.noStateCheck = true } +} + +// WithMaxSideEffectLag bounds how long StepPure may leave a side-effect task +// (e.g. InvokerExecuteTask) undispatched while moving the clock forward. +func WithMaxSideEffectLag(lag time.Duration) DriverOption { + return func(c *driverConfig) { c.maxLag = lag } +} + +// WithMaxRunDuration bounds how long a started workflow runs: the driver +// completes it before moving the clock past StartTime plus d. +func WithMaxRunDuration(d time.Duration) DriverOption { + return func(c *driverConfig) { c.maxRun = d } +} + +// NewDriver constructs a Driver and starts a scheduler execution via the +// production CreateScheduler path. startTime defaults to a fixed instant for +// determinism. +func NewDriver(t chasmtest.TB, opts ...DriverOption) *Driver { + t.Helper() + d, cfg := newDriverScaffold(t, opts...) + req := &schedulerpb.CreateScheduleRequest{ + NamespaceId: NamespaceID, + FrontendRequest: &workflowservice.CreateScheduleRequest{ + Namespace: Namespace, + ScheduleId: ScheduleID, + Schedule: cfg.schedule, + RequestId: CreateRequestID, + }, + } + d.start(func(mctx chasm.MutableContext) (*scheduler.Scheduler, error) { + return scheduler.CreateScheduler(mctx, req) + }) + return d +} + +// NewDriverFromMigration constructs a Driver and starts a scheduler execution +// from a migrated V1 state via the production CreateSchedulerFromMigration path. +// This is the V1->V2 import side of a migration round trip. +func NewDriverFromMigration(t chasmtest.TB, req *schedulerpb.CreateFromMigrationStateRequest, opts ...DriverOption) *Driver { + t.Helper() + d, _ := newDriverScaffold(t, opts...) + d.start(func(mctx chasm.MutableContext) (*scheduler.Scheduler, error) { + return scheduler.CreateSchedulerFromMigration(mctx, req) + }) + return d +} + +// newDriverScaffold builds the engine, fake services and clock, but does not +// start an execution. +func newDriverScaffold(t chasmtest.TB, opts ...DriverOption) (*Driver, *driverConfig) { + t.Helper() + + cfg := &driverConfig{ + schedule: DefaultSchedule(), + startTime: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC), + config: DefaultConfig(), + } + for _, opt := range opts { + opt(cfg) + } + + ts := clock.NewEventTimeSource() + ts.Update(cfg.startTime) + services := newFakeServices(t, gomock.NewController(t), ts, cfg.controlledStarts) + metricsHandler := metricstest.NewCaptureHandler() + + engineOpts := []Option{ + WithLogger(testlogger.NewTestLogger(t, testlogger.FailOnExpectedErrorOnly)), + WithSchedulerConfig(cfg.config), + WithTimeSource(ts), + WithInvokerTimeSource(ts), + WithInvokerMetrics(metricsHandler), + WithClients(services.Frontend, services.History), + } + if cfg.noStateCheck { + engineOpts = append(engineOpts, WithoutStateCheck()) + } + + d := &Driver{ + SchedulerEngine: NewEngine(t, engineOpts...), + fakeServices: services, + recordCalls: make(chan *RecordCall), + recorded: make(map[string]ExternalStart), + replacingSeen: make(map[string]struct{}), + metricsCapture: metricsHandler.StartCapture(), + maxRunDuration: cfg.maxRun, + AfterStep: cfg.afterStep, + } + d.runner = d.newTaskRunner(cfg.maxLag) + return d, cfg +} + +// start starts the scheduler execution using the given start function. +func (d *Driver) start(startFn func(chasm.MutableContext) (*scheduler.Scheduler, error)) { + d.T.Helper() + if err := d.Start(startFn); err != nil { + d.T.Fatalf("failed to start scheduler execution: %v", err) + } + d.ReadScheduler(func(s *scheduler.Scheduler, _ chasm.Context) { d.initialActionCount = s.Info.GetActionCount() }) +} + +// Now returns the engine's current clock time. +func (d *Driver) Now() time.Time { + return d.TimeSource.Now() +} + +// ReadScheduler reads the scheduler root component and invokes fn with it. +func (d *Driver) ReadScheduler(fn func(s *scheduler.Scheduler, ctx chasm.Context)) { + d.T.Helper() + err := d.Read(func(s *scheduler.Scheduler, ctx chasm.Context) error { + fn(s, ctx) + return nil + }) + if err != nil { + d.T.Fatalf("ReadScheduler failed: %v", err) + } +} + +// Step advances the clock to the next scheduled task and fires all tasks due at +// that time: pure tasks first, then every due side-effect task. It returns false +// if nothing was pending (the schedule is quiescent). +func (d *Driver) Step() bool { + d.T.Helper() + return d.runner.Step() +} + +// StepPure advances the clock to the next pending pure task and fires every pure +// task due then, leaving side-effect tasks undispatched. This models +// InvokerExecuteTask lagging behind the scheduler's timers. It returns false if +// no pure task is pending, or if firing it would move the clock more than +// the configured max side-effect lag past the oldest undispatched side-effect task. +func (d *Driver) StepPure() bool { + d.T.Helper() + return d.runner.StepPure() +} + +// StepSideEffect dispatches the oldest side-effect task that is due at the +// current time, without moving the clock. It returns false if none is due. +func (d *Driver) StepSideEffect() bool { + d.T.Helper() + return d.runner.StepSideEffect() +} + +// RunToQuiescence steps until no task is pending or maxSteps is reached. It +// returns the number of steps taken and whether the schedule reached quiescence +// (no pending task). A normal running schedule never quiesces and will stop at +// maxSteps with quiescent == false. +func (d *Driver) RunToQuiescence(maxSteps int) (steps int, quiescent bool) { + d.T.Helper() + return d.runner.RunToQuiescence(maxSteps) +} + +// HasPendingTask reports whether any task (pure or side-effect) is currently +// scheduled to fire. +func (d *Driver) HasPendingTask() bool { + d.T.Helper() + return d.runner.HasPendingTask() +} + +// newTaskRunner wires the scheduler-specific behaviour into a generic task +// runner. +func (d *Driver) newTaskRunner(maxLag time.Duration) *chasmtest.TaskRunner { + return chasmtest.NewTaskRunner(d.T, d.Engine, d.Ref, d.TimeSource, + chasmtest.WithMaxSideEffectLag(maxLag), + chasmtest.WithAsyncEngine(func(engine chasm.Engine) chasm.Engine { + return &recordingEngine{Engine: engine, calls: d.recordCalls} + }), + chasmtest.WithTaskRunnerHooks(chasmtest.TaskRunnerHooks{ + BeforeAdvance: d.completeWorkflowsBefore, + BeforeSideEffect: func() { d.actionsBefore = d.actionCount() }, + // A side-effect task may find the schedule already closed and gone. + TolerateSideEffectError: func(err error) bool { + _, ok := errors.AsType[*serviceerror.NotFound](err) + return ok + }, + AfterSideEffect: func() { d.maxActionsObserved = max(d.maxActionsObserved, d.actionCount()-d.actionsBefore) }, + AfterSideEffectCommit: d.observeRecordedStarts, + AfterStep: d.runAfterStep, + }), + ) +} + +func (d *Driver) runAfterStep() { + if d.AfterStep != nil { + d.AfterStep(d) + } +} + +// completeWorkflowsBefore completes any workflow that would exceed +// maxRunDuration before the clock moves to t, moving the clock to the deadline +// first. +func (d *Driver) completeWorkflowsBefore(t time.Time) { + for d.maxRunDuration > 0 { + startTime, ok := d.oldestRunningStartTime() + if !ok { + break + } + due := startTime.Add(d.maxRunDuration) + if due.After(t) { + break + } + if due.After(d.Now()) { + d.TimeSource.Update(due) + } + d.CompleteWorkflow() + // The completion may ready a deferred start; don't skip its execute task. + for d.StepSideEffect() { + } + } +} + +// oldestRunningStartTime returns the StartTime of the oldest started workflow +// that hasn't completed. +func (d *Driver) oldestRunningStartTime() (time.Time, bool) { + var oldest time.Time + have := false + d.ReadScheduler(func(s *scheduler.Scheduler, ctx chasm.Context) { + if s.Closed || s.IsSentinel() { + return + } + if start := oldestRunning(s.Invoker.Get(ctx).GetBufferedStarts()); start != nil { + oldest, have = start.GetStartTime().AsTime(), true + } + }) + return oldest, have +} + +// oldestRunning returns the earliest-started buffered start that has a run but +// has not completed, or nil. +func oldestRunning(starts []*schedulespb.BufferedStart) *schedulespb.BufferedStart { + var oldest *schedulespb.BufferedStart + for _, start := range starts { + if start.GetRunId() == "" || start.GetCompleted() != nil { + continue + } + if oldest == nil || start.GetStartTime().AsTime().Before(oldest.GetStartTime().AsTime()) { + oldest = start + } + } + return oldest +} + +// StateViolations evaluates the shared stuck and idle-deadline invariants +// against the last committed state without failing the test. See +// [WithoutStateInvariant]. +func (d *Driver) StateViolations() []string { + d.T.Helper() + return d.SchedulerEngine.StateViolations() +} + +// apply runs op against the scheduler root in a single transaction. On success +// the AfterStep hook runs, so invariants are checked at the new settled state. +// It returns the operation's error without failing the test, so callers driving +// random operations can tolerate rejected ones (e.g. ErrClosed once the +// schedule has closed). +func (d *Driver) apply(op func(*scheduler.Scheduler, chasm.MutableContext) error) error { + d.T.Helper() + err := d.SchedulerEngine.Update(op) + if err == nil { + d.runAfterStep() + } + return err +} + +// CompleteWorkflow delivers a successful completion callback for the oldest +// running workflow, through Scheduler.HandleNexusCompletion. It returns false +// if no workflow is running. +func (d *Driver) CompleteWorkflow() bool { + d.T.Helper() + completed := false + err := d.SchedulerEngine.Update(func(s *scheduler.Scheduler, mctx chasm.MutableContext) error { + if s.Closed || s.IsSentinel() { + return nil + } + oldest := oldestRunning(s.Invoker.Get(mctx).GetBufferedStarts()) + if oldest == nil { + return nil + } + completed = true + return s.HandleNexusCompletion(mctx, &persistencespb.ChasmNexusCompletion{ + RequestId: oldest.GetRequestId(), + Outcome: &persistencespb.ChasmNexusCompletion_Success{Success: &commonpb.Payload{}}, + CloseTime: timestamppb.New(d.Now()), + }) + }) + if err != nil { + d.T.Fatalf("CompleteWorkflow failed: %v", err) + } + if completed { + d.runAfterStep() + } + return completed +} + +// DeliverCompletion delivers a completion callback for requestID. It also +// supports accepted starts whose RPC results have not been recorded locally, and +// duplicate or obsolete callback request identities. +func (d *Driver) DeliverCompletion(requestID string) error { + return d.SchedulerEngine.Update(func(s *scheduler.Scheduler, mctx chasm.MutableContext) error { + return s.HandleNexusCompletion(mctx, &persistencespb.ChasmNexusCompletion{ + RequestId: requestID, CloseTime: timestamppb.New(d.Now()), + Outcome: &persistencespb.ChasmNexusCompletion_Success{Success: &commonpb.Payload{}}, + }) + }) +} + +// Patch applies a SchedulePatch through the production Scheduler.Patch path in a +// single transaction. It returns the scheduler's error (e.g. ErrClosed when the +// schedule has already closed) without failing the test. On success the +// AfterStep hook runs. +func (d *Driver) Patch(patch *schedulepb.SchedulePatch) error { + d.T.Helper() + return d.apply(func(s *scheduler.Scheduler, mctx chasm.MutableContext) error { + _, err := s.Patch(mctx, &schedulerpb.PatchScheduleRequest{ + NamespaceId: NamespaceID, + FrontendRequest: &workflowservice.PatchScheduleRequest{ + Namespace: Namespace, + ScheduleId: ScheduleID, + Patch: patch, + }, + }) + return err + }) +} + +// Update applies mutate to a copy of the current schedule definition and +// replaces the schedule with it through the production Scheduler.Update path, +// without a conflict token. It returns the scheduler's error (e.g. ErrClosed) +// without failing the test. On success the AfterStep hook runs. (The embedded +// SchedulerEngine.Update is the lower-level typed helper.) +func (d *Driver) Update(mutate func(*schedulepb.Schedule)) error { + d.T.Helper() + return d.apply(func(s *scheduler.Scheduler, mctx chasm.MutableContext) error { + next := common.CloneProto(s.Schedule) + mutate(next) + _, err := s.Update(mctx, &schedulerpb.UpdateScheduleRequest{ + NamespaceId: NamespaceID, + FrontendRequest: &workflowservice.UpdateScheduleRequest{ + Namespace: Namespace, + ScheduleId: ScheduleID, + Schedule: next, + }, + }) + return err + }) +} + +// Backfill requests a backfill of the given range via Patch. +func (d *Driver) Backfill(req *schedulepb.BackfillRequest) error { + return d.Patch(&schedulepb.SchedulePatch{BackfillRequest: []*schedulepb.BackfillRequest{req}}) +} + +// Pause pauses the schedule via Patch. +func (d *Driver) Pause() error { + return d.Patch(&schedulepb.SchedulePatch{Pause: "paused by driver"}) +} + +// Unpause unpauses the schedule via Patch. +func (d *Driver) Unpause() error { + return d.Patch(&schedulepb.SchedulePatch{Unpause: "unpaused by driver"}) +} + +// TriggerImmediately requests an immediate action via Patch. +func (d *Driver) TriggerImmediately() error { + return d.Patch(&schedulepb.SchedulePatch{ + TriggerImmediately: &schedulepb.TriggerImmediatelyRequest{}, + }) +} + +// MigrateToWorkflow initiates a V2->V1 migration: it pauses the schedule and +// schedules the SchedulerMigrateToWorkflowTask side-effect task. Step the driver +// afterward to fire the task (which calls the history client's +// StartWorkflowExecution with the exported StartScheduleArgs). It does not run +// the AfterStep hook. +func (d *Driver) MigrateToWorkflow() error { + d.T.Helper() + return d.SchedulerEngine.Update(func(s *scheduler.Scheduler, mctx chasm.MutableContext) error { + _, err := s.MigrateToWorkflow(mctx, &schedulerpb.MigrateToWorkflowRequest{ + NamespaceId: NamespaceID, + ScheduleId: ScheduleID, + }) + return err + }) +} diff --git a/chasm/lib/scheduler/schedulertest/driver_smoke_test.go b/chasm/lib/scheduler/schedulertest/driver_smoke_test.go new file mode 100644 index 00000000000..d3d52d6d002 --- /dev/null +++ b/chasm/lib/scheduler/schedulertest/driver_smoke_test.go @@ -0,0 +1,78 @@ +package schedulertest_test + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/chasmtest" + "go.temporal.io/server/chasm/lib/scheduler" + "go.temporal.io/server/chasm/lib/scheduler/schedulertest" + "pgregory.net/rapid" +) + +// TestDriver_RunsIntervalScheduleForward verifies the driver fires the +// scheduler's self-scheduled tasks and advances the clock across several +// interval ticks of a normal (running) schedule. +func TestDriver_RunsIntervalScheduleForward(t *testing.T) { + start := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) + d := schedulertest.NewDriver(t, schedulertest.WithStartTime(start)) + + steps, quiescent := d.RunToQuiescence(20) + + // A normal interval schedule re-arms a generator task forever, so it never + // quiesces; it should burn the whole step budget. + require.False(t, quiescent, "running interval schedule should not quiesce") + require.Equal(t, 20, steps) + + // The clock must have advanced as interval ticks fire. + require.True(t, d.Now().After(start.Add(schedulertest.DefaultInterval)), + "clock should advance past one interval, now=%s", d.Now()) + + d.ReadScheduler(func(s *scheduler.Scheduler, _ chasm.Context) { + require.False(t, s.Closed, "schedule should still be open") + }) +} + +// TestDriver_PausedScheduleKeepsTicking documents that a paused interval +// schedule is *held open* and keeps re-arming its generator task to advance the +// high-water mark — it neither quiesces nor closes — but it never buffers a +// start. +func TestDriver_PausedScheduleKeepsTicking(t *testing.T) { + sched := schedulertest.DefaultSchedule() + sched.State.Paused = true + + d := schedulertest.NewDriver(t, schedulertest.WithSchedule(sched)) + + _, quiescent := d.RunToQuiescence(10) + require.False(t, quiescent, "paused interval schedule keeps ticking, not quiescent") + + d.ReadScheduler(func(s *scheduler.Scheduler, _ chasm.Context) { + require.True(t, s.Schedule.State.Paused) + require.False(t, s.Closed, "paused schedule is held open, not closed") + }) +} + +// TestDriver_LimitedActionsExhaustedClosesAndQuiesces verifies that a schedule +// with no remaining actions arms an idle task instead of generating work, and +// after the idle task fires the schedule closes and the driver reaches +// quiescence. +func TestDriver_LimitedActionsExhaustedClosesAndQuiesces(t *testing.T) { + sched := schedulertest.DefaultSchedule() + sched.State.LimitedActions = true + sched.State.RemainingActions = 0 + + d := schedulertest.NewDriver(t, schedulertest.WithSchedule(sched)) + + _, quiescent := d.RunToQuiescence(10) + require.True(t, quiescent, "exhausted schedule should close and quiesce") + + d.ReadScheduler(func(s *scheduler.Scheduler, _ chasm.Context) { + require.True(t, s.Closed, "schedule should be closed after idle task fires") + }) +} + +// *rapid.T must stay usable as the driver's test handle: harness failures then +// flow through rapid's shrinking. +var _ chasmtest.TB = (*rapid.T)(nil) diff --git a/chasm/lib/scheduler/schedulertest/engine.go b/chasm/lib/scheduler/schedulertest/engine.go new file mode 100644 index 00000000000..cdf968b3036 --- /dev/null +++ b/chasm/lib/scheduler/schedulertest/engine.go @@ -0,0 +1,244 @@ +package schedulertest + +import ( + "context" + + "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/chasmtest" + "go.temporal.io/server/chasm/lib/scheduler" + "go.temporal.io/server/common/clock" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/resource" + "go.temporal.io/server/common/testing/testlogger" + legacyscheduler "go.temporal.io/server/service/worker/scheduler" + "go.uber.org/mock/gomock" +) + +// CreateRequestID is the request ID [SchedulerEngine.Start] records for the +// scheduler's creating transaction. +const CreateRequestID = "create-req" + +// SchedulerEngine bundles a [chasmtest.Engine] with the CHASM registry that has +// the scheduler library registered, the reference of the Scheduler root +// component, and typed read/update helpers. It is the shared base of the unit +// tests in package scheduler_test and of [Driver]. +// +// NewEngine only builds the engine. Create the execution with [SchedulerEngine.Start], +// or through a handler using [SchedulerEngine.Ctx]. +type SchedulerEngine struct { + T chasmtest.TB + Engine *chasmtest.Engine + Registry *chasm.Registry + Logger log.Logger + // State is the stuck and idle-deadline invariant the engine asserts after + // every committed transition (unless built WithoutStateCheck). + State *StateInvariant + // Config is the scheduler config the library was wired with. + Config *scheduler.Config + // TimeSource is the clock the engine was built with, or nil if the engine + // kept its default. + TimeSource *clock.EventTimeSource + // SpecProcessor is the processor the library was wired with. + SpecProcessor scheduler.SpecProcessor + // Ctx carries the engine, ready for chasm.StartExecution, chasm.ReadComponent + // and handler calls. + Ctx context.Context + Key chasm.ExecutionKey + Ref chasm.ComponentRef + + // node and root are the tree and Scheduler as of the last committed + // transition, for StateViolations. + node *chasm.Node + root *scheduler.Scheduler +} + +type engineOptions struct { + logger log.Logger + library LibraryOptions + engineOpts []chasmtest.EngineOption + timeSource *clock.EventTimeSource + noStateCheck bool +} + +// Option configures [NewEngine]. +type Option func(*engineOptions) + +// WithLogger overrides the default test logger. +func WithLogger(l log.Logger) Option { + return func(o *engineOptions) { o.logger = l } +} + +// WithSchedulerConfig overrides the scheduler config (default: [DefaultConfig]). Use it +// to set small Tweakables, or a different retry policy. +func WithSchedulerConfig(cfg *scheduler.Config) Option { + return func(o *engineOptions) { o.library.Config = cfg } +} + +// WithSpecProcessor overrides the real spec processor, e.g. to simulate failures. +func WithSpecProcessor(sp scheduler.SpecProcessor) Option { + return func(o *engineOptions) { o.library.SpecProcessor = sp } +} + +// WithSpecBuilder overrides the spec builder used by both the library and the +// default spec processor (default: [DefaultSpecBuilder]). +func WithSpecBuilder(b *legacyscheduler.SpecBuilder) Option { + return func(o *engineOptions) { o.library.SpecBuilder = b } +} + +// WithTimeSource gives the engine a controllable clock. Without it the engine +// keeps chasmtest's default, which is frozen at time.Now. +func WithTimeSource(ts *clock.EventTimeSource) Option { + return func(o *engineOptions) { + o.timeSource = ts + o.engineOpts = append(o.engineOpts, chasmtest.WithTimeSource(ts)) + } +} + +// WithInvokerTimeSource also hands the clock to the invoker, which stamps +// StartTime on started workflows. Without it the invoker uses real time. +func WithInvokerTimeSource(ts clock.TimeSource) Option { + return func(o *engineOptions) { o.library.TimeSource = ts } +} + +// WithInvokerMetrics gives the invoker task handlers a metrics handler. +func WithInvokerMetrics(h metrics.Handler) Option { + return func(o *engineOptions) { o.library.MetricsHandler = h } +} + +// WithClients wires the frontend and history clients behind the side-effect +// task handlers (execute, callbacks, migrate). +func WithClients(frontend workflowservice.WorkflowServiceClient, history resource.HistoryClient) Option { + return func(o *engineOptions) { + o.library.FrontendClient = frontend + o.library.HistoryClient = history + } +} + +// WithEngineOptions passes options straight to [chasmtest.NewEngine], e.g. an +// extra invariant check or an engine metrics handler. +func WithEngineOptions(opts ...chasmtest.EngineOption) Option { + return func(o *engineOptions) { o.engineOpts = append(o.engineOpts, opts...) } +} + +// WithoutStateCheck leaves the stuck and idle-deadline invariants off the +// engine. Only a test that corrupts the scheduler on purpose and asserts +// [StateInvariant.Violations] catches it should need this. +func WithoutStateCheck() Option { + return func(o *engineOptions) { o.noStateCheck = true } +} + +// NewEngine builds a registry with the core and scheduler libraries, wraps it in +// a [chasmtest.Engine], and returns the wrapper. No execution is started. +func NewEngine(t chasmtest.TB, opts ...Option) *SchedulerEngine { + t.Helper() + + o := &engineOptions{} + for _, opt := range opts { + opt(o) + } + if o.logger == nil { + o.logger = testlogger.NewTestLogger(t, testlogger.FailOnExpectedErrorOnly) + } + if o.library.Config == nil { + o.library.Config = DefaultConfig() + } + if o.library.SpecBuilder == nil { + o.library.SpecBuilder = DefaultSpecBuilder() + } + if o.library.SpecProcessor == nil { + o.library.SpecProcessor = NewSpecProcessor(gomock.NewController(t), o.logger, o.library.SpecBuilder) + } + o.library.Logger = o.logger + + registry := chasm.NewRegistry(o.logger) + if err := registry.Register(&chasm.CoreLibrary{}); err != nil { + t.Fatalf("failed to register core library: %v", err) + } + if err := registry.Register(NewTestLibrary(o.library)); err != nil { + t.Fatalf("failed to register scheduler library: %v", err) + } + + state := NewStateInvariant(registry, o.library.Config, o.logger) + se := &SchedulerEngine{ + T: t, + Registry: registry, + State: state, + Logger: o.logger, + Config: o.library.Config, + TimeSource: o.timeSource, + SpecProcessor: o.library.SpecProcessor, + } + engineOpts := append(o.engineOpts, chasmtest.WithInvariantCheck( + func(t chasmtest.TB, node *chasm.Node, root chasm.RootComponent) { + t.Helper() + sched, ok := root.(*scheduler.Scheduler) + if !ok { + t.Fatalf("scheduler test engine root has unexpected type %T", root) + } + se.node, se.root = node, sched + if !o.noStateCheck { + state.Check(t, node, sched) + } + }, + )) + engine := chasmtest.NewEngine(t, registry, engineOpts...) + + se.Engine = engine + se.Key = chasm.ExecutionKey{NamespaceID: NamespaceID, BusinessID: ScheduleID} + se.Ref = chasm.NewComponentRef[*scheduler.Scheduler](se.Key) + se.Ctx = chasm.NewEngineContext(context.Background(), engine) + return se +} + +// StateViolations evaluates the stuck and idle-deadline invariants against the +// last committed state without failing the test. It is how a test built +// [WithoutStateCheck] asserts that the check catches state it corrupted on +// purpose. +func (e *SchedulerEngine) StateViolations() []string { + e.T.Helper() + if e.node == nil { + e.T.Fatalf("no transition has committed yet") + } + return e.State.Violations(e.node, e.root) +} + +// Start starts the scheduler execution with startFn, recording [CreateRequestID]. +func (e *SchedulerEngine) Start(startFn func(chasm.MutableContext) (*scheduler.Scheduler, error)) error { + e.T.Helper() + _, err := chasm.StartExecution(e.Ctx, e.Key, func(mctx chasm.MutableContext, _ *struct{}) (*scheduler.Scheduler, error) { + return startFn(mctx) + }, (*struct{})(nil), chasm.WithRequestID(CreateRequestID)) + return err +} + +// Update runs update against the Scheduler root component through the engine's +// UpdateComponent path. +func (e *SchedulerEngine) Update(update func(*scheduler.Scheduler, chasm.MutableContext) error) error { + e.T.Helper() + _, _, err := chasm.UpdateComponent( + e.Ctx, + e.Ref, + func(s *scheduler.Scheduler, ctx chasm.MutableContext, _ struct{}) (struct{}, error) { + return struct{}{}, update(s, ctx) + }, + struct{}{}, + ) + return err +} + +// Read runs read against the Scheduler root component through the engine's +// read-only ReadComponent path. +func (e *SchedulerEngine) Read(read func(*scheduler.Scheduler, chasm.Context) error) error { + e.T.Helper() + _, err := chasm.ReadComponent( + e.Ctx, + e.Ref, + func(s *scheduler.Scheduler, ctx chasm.Context, _ struct{}) (struct{}, error) { + return struct{}{}, read(s, ctx) + }, + struct{}{}, + ) + return err +} diff --git a/chasm/lib/scheduler/schedulertest/external.go b/chasm/lib/scheduler/schedulertest/external.go new file mode 100644 index 00000000000..9f33968b4cc --- /dev/null +++ b/chasm/lib/scheduler/schedulertest/external.go @@ -0,0 +1,201 @@ +package schedulertest + +import ( + "context" + "fmt" + "maps" + "sync" + + "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/chasmtest" + "go.temporal.io/server/chasm/lib/scheduler" +) + +// ExternalStart records acceptance outside the scheduler's transaction. +type ExternalStart struct { + WorkflowID string + RunID string +} + +type externalLedger struct { + mu sync.Mutex + starts map[string]ExternalStart +} + +func (l *externalLedger) accept(req *workflowservice.StartWorkflowExecutionRequest) (ExternalStart, error) { + l.mu.Lock() + defer l.mu.Unlock() + if start, ok := l.starts[req.GetRequestId()]; ok { + if start.WorkflowID != req.GetWorkflowId() { + return ExternalStart{}, fmt.Errorf("request %s changed workflow identity", req.GetRequestId()) + } + return start, nil + } + start := ExternalStart{WorkflowID: req.GetWorkflowId(), RunID: "run-" + req.GetRequestId()} + l.starts[req.GetRequestId()] = start + return start, nil +} + +func (l *externalLedger) snapshot() map[string]ExternalStart { + l.mu.Lock() + defer l.mu.Unlock() + return maps.Clone(l.starts) +} + +// StartCall is an outstanding RPC. Accept and Respond are distinct events so a +// test can deliver a callback or lose the response after external acceptance. +type StartCall struct { + Request *workflowservice.StartWorkflowExecutionRequest + t chasmtest.TB + result chan startResponse + ledger *externalLedger +} + +type startResponse struct { + response *workflowservice.StartWorkflowExecutionResponse + err error +} + +func (c *StartCall) Accept() (ExternalStart, error) { return c.ledger.accept(c.Request) } + +func (c *StartCall) Respond(start ExternalStart, err error) { + select { + case c.result <- startResponse{response: &workflowservice.StartWorkflowExecutionResponse{RunId: start.RunID}, err: err}: + default: + c.t.Fatal("StartCall.Respond called more than once") + } +} + +// WithControlledStarts makes start RPCs wait for explicit Accept/Respond events. +func WithControlledStarts() DriverOption { + return func(c *driverConfig) { c.controlledStarts = true } +} + +// BeginSideEffect runs until the handler blocks at a start RPC or finishes. +// FinishSideEffect must be called before starting another task. Engine access +// between these calls is allowed only while all start RPCs are blocked or the +// handler is waiting at NextRecord. +func (d *Driver) BeginSideEffect() bool { + d.T.Helper() + if d.startCalls == nil { + d.T.Fatal("controlled starts required") + } + return d.runner.BeginSideEffect() +} + +func (d *Driver) NextStart() *StartCall { + d.T.Helper() + select { + case call := <-d.startCalls: + return call + case err := <-d.runner.SideEffectDone(): + d.runner.AbandonSideEffect() + d.T.Fatalf("task finished before expected start: %v", err) + case <-d.T.Context().Done(): + d.T.Fatal("waiting for start: test canceled") + } + return nil +} + +func (d *Driver) FinishSideEffect() { + d.T.Helper() + for { + select { + case call := <-d.recordCalls: + if err := call.Record(); err != nil { + d.T.Fatalf("record start result: %v", err) + } + case call := <-d.startCalls: + d.T.Fatalf("unconsumed start RPC for request %s; call NextStart before FinishSideEffect", call.Request.GetRequestId()) + case err := <-d.runner.SideEffectDone(): + d.runner.FinishSideEffect(err) + return + case <-d.T.Context().Done(): + d.T.Fatal("waiting for side effect: test canceled") + } + } +} + +// NextRecord waits until responses have been collected and the handler is ready +// to record them. The caller can deliver other events before calling Record. +func (d *Driver) NextRecord() *RecordCall { + d.T.Helper() + select { + case call := <-d.recordCalls: + return call + case err := <-d.runner.SideEffectDone(): + d.runner.AbandonSideEffect() + d.T.Fatalf("task finished before result recording: %v", err) + case <-d.T.Context().Done(): + d.T.Fatal("waiting for recording: test canceled") + } + return nil +} + +type recordingEngine struct { + chasm.Engine + calls chan *RecordCall +} + +// RecordCall separates successful RPC response delivery from local recording. +// It uses the actual handler's update closure; it is not a durable storage commit. +type RecordCall struct { + engine chasm.Engine + ctx context.Context + ref chasm.ComponentRef + update func(chasm.MutableContext, chasm.Component) error + opts []chasm.TransitionOption + result chan recordResponse +} + +type recordResponse struct { + ref []byte + err error +} + +func (e *recordingEngine) UpdateComponent(ctx context.Context, ref chasm.ComponentRef, + update func(chasm.MutableContext, chasm.Component) error, opts ...chasm.TransitionOption) ([]byte, error) { + call := &RecordCall{engine: e.Engine, ctx: ctx, ref: ref, update: update, opts: opts, result: make(chan recordResponse, 1)} + select { + case e.calls <- call: + case <-ctx.Done(): + return nil, ctx.Err() + } + select { + case result := <-call.result: + return result.ref, result.err + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +func (c *RecordCall) Record() error { + ref, err := c.engine.UpdateComponent(c.ctx, c.ref, c.update, c.opts...) + c.result <- recordResponse{ref: ref, err: err} + return err +} + +func (d *Driver) observeRecordedStarts() { + accepted := d.external.snapshot() + d.ReadScheduler(func(s *scheduler.Scheduler, ctx chasm.Context) { + if s.IsSentinel() { + return + } + for _, start := range s.Invoker.Get(ctx).GetBufferedStarts() { + if start.GetRunId() != "" { + if _, ok := accepted[start.GetRequestId()]; ok { + d.recorded[start.GetRequestId()] = ExternalStart{WorkflowID: start.GetWorkflowId(), RunID: start.GetRunId()} + } + } + } + for _, result := range s.Info.GetRecentActions() { + execution := result.GetStartWorkflowResult() + for id, start := range accepted { + if execution.GetWorkflowId() == start.WorkflowID && execution.GetRunId() == start.RunID { + d.recorded[id] = start + } + } + } + }) +} diff --git a/chasm/lib/scheduler/schedulertest/external_test.go b/chasm/lib/scheduler/schedulertest/external_test.go new file mode 100644 index 00000000000..201820a5f58 --- /dev/null +++ b/chasm/lib/scheduler/schedulertest/external_test.go @@ -0,0 +1,137 @@ +package schedulertest_test + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + enumspb "go.temporal.io/api/enums/v1" + "go.temporal.io/api/serviceerror" + "go.temporal.io/server/chasm/lib/scheduler" + "go.temporal.io/server/chasm/lib/scheduler/schedulertest" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestStartAcceptanceCallbackAndRecording(t *testing.T) { + t.Parallel() + for _, policy := range []enumspb.ScheduleOverlapPolicy{ + enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL, + enumspb.SCHEDULE_OVERLAP_POLICY_SKIP, + enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ALL, + } { + for _, position := range []string{"before_response", "before_record", "after_record"} { + t.Run(policy.String()+"/"+position, func(t *testing.T) { + t.Parallel() + start := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) + schedule := oneShotSchedule(start, time.Minute) + schedule.Policies.OverlapPolicy = policy + d := schedulertest.NewDriver(t, schedulertest.WithSchedule(schedule), schedulertest.WithControlledStarts()) + require.True(t, d.StepPure()) + require.True(t, d.BeginSideEffect()) + call := d.NextStart() + accepted, err := call.Accept() + require.NoError(t, err) + before := d.Snapshot() + require.Len(t, before.AcceptedStarts, 1) + require.Empty(t, before.RecordedStarts) + require.Zero(t, before.ActionCount) + if position == "before_response" { + require.NoError(t, d.DeliverCompletion(call.Request.GetRequestId())) + require.NoError(t, d.DeliverCompletion(call.Request.GetRequestId())) + } + call.Respond(accepted, nil) + record := d.NextRecord() + require.Zero(t, d.Snapshot().ActionCount, "a successful response is not yet a recorded start") + if position == "before_record" { + require.NoError(t, d.DeliverCompletion(call.Request.GetRequestId())) + require.NoError(t, d.DeliverCompletion(call.Request.GetRequestId())) + } + require.NoError(t, record.Record()) + d.FinishSideEffect() + if position == "after_record" { + require.NoError(t, d.DeliverCompletion(call.Request.GetRequestId())) + require.NoError(t, d.DeliverCompletion(call.Request.GetRequestId())) + } + cur := d.Snapshot() + require.Empty(t, schedulertest.CheckInvariants(&before, cur)) + require.Equal(t, int64(1), cur.ActionCount) + require.Equal(t, accepted, cur.RecordedStarts[call.Request.GetRequestId()]) + }) + } + } +} + +func TestLostStartResponseDeduplicatesRetry(t *testing.T) { + t.Parallel() + start := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) + schedule := oneShotSchedule(start, time.Minute) + schedule.Policies.OverlapPolicy = enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL + d := schedulertest.NewDriver(t, schedulertest.WithSchedule(schedule), schedulertest.WithControlledStarts()) + require.True(t, d.StepPure()) + require.True(t, d.BeginSideEffect()) + first := d.NextStart() + accepted, err := first.Accept() + require.NoError(t, err) + first.Respond(accepted, serviceerror.NewUnavailable("accepted but response lost")) + d.FinishSideEffect() + pending := d.Snapshot() + require.Len(t, pending.AcceptedStarts, 1) + require.Len(t, pending.PendingStarts, 1) + require.Zero(t, pending.ActionCount) + require.Empty(t, schedulertest.CheckInvariants(nil, pending)) + require.True(t, d.StepPure()) + require.True(t, d.BeginSideEffect()) + retry := d.NextStart() + require.Equal(t, first.Request.GetRequestId(), retry.Request.GetRequestId()) + require.Equal(t, first.Request.GetWorkflowId(), retry.Request.GetWorkflowId()) + again, err := retry.Accept() + require.NoError(t, err) + require.Equal(t, accepted, again) + retry.Respond(again, nil) + d.FinishSideEffect() + cur := d.Snapshot() + require.Len(t, cur.AcceptedStarts, 1) + require.Equal(t, int64(1), cur.ActionCount) + require.Empty(t, schedulertest.CheckInvariants(&pending, cur)) +} + +func TestAcceptedStartAccountingDetector(t *testing.T) { + t.Parallel() + start := schedulertest.ExternalStart{WorkflowID: "workflow", RunID: "run"} + cases := []struct { + name string + snapshot schedulertest.Snapshot + want string + }{ + {"lost after RPC", schedulertest.Snapshot{AcceptedStarts: map[string]schedulertest.ExternalStart{"a": start}}, "accepted-start-lost"}, + {"double counted", schedulertest.Snapshot{AcceptedStarts: map[string]schedulertest.ExternalStart{"a": start}, RecordedStarts: map[string]schedulertest.ExternalStart{"a": start}, ActionCount: 2}, "start-accounting"}, + {"wrong run", schedulertest.Snapshot{AcceptedStarts: map[string]schedulertest.ExternalStart{"a": start}, RecordedStarts: map[string]schedulertest.ExternalStart{"a": {WorkflowID: "workflow", RunID: "wrong"}}, ActionCount: 1}, "start-identity"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + tc.snapshot.HasPendingTask = true + require.Contains(t, violationNames(schedulertest.CheckInvariants(nil, tc.snapshot)), tc.want) + }) + } +} + +func TestExpiredSelectionReleasesRemainingBuffer(t *testing.T) { + t.Parallel() + start := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) + schedule := schedulertest.DefaultSchedule() + schedule.Spec.Interval[0].Interval = durationpb.New(10 * time.Second) + schedule.Spec.EndTime = timestamppb.New(start.Add(31 * time.Second)) + schedule.Policies.CatchupWindow = durationpb.New(time.Minute) + schedule.Policies.OverlapPolicy = enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ALL + d := schedulertest.NewDriver(t, schedulertest.WithSchedule(schedule), + schedulertest.WithMaxRunDuration(5*time.Minute), + schedulertest.WithConfig(configWith(func(tw *scheduler.Tweakables) { tw.IdleTime = 10 * time.Minute })), + schedulertest.WithAfterStep(schedulertest.CheckInvariantsHook(t))) + for range 3 { + require.True(t, d.Step()) + } + require.True(t, d.StepPure()) + require.Empty(t, d.Snapshot().PendingStarts) +} diff --git a/chasm/lib/scheduler/schedulertest/fakes.go b/chasm/lib/scheduler/schedulertest/fakes.go new file mode 100644 index 00000000000..51227508fe6 --- /dev/null +++ b/chasm/lib/scheduler/schedulertest/fakes.go @@ -0,0 +1,173 @@ +package schedulertest + +import ( + "context" + "maps" + "sync" + + enumspb "go.temporal.io/api/enums/v1" + "go.temporal.io/api/serviceerror" + workflowpb "go.temporal.io/api/workflow/v1" + "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/server/api/historyservice/v1" + "go.temporal.io/server/api/historyservicemock/v1" + "go.temporal.io/server/chasm/chasmtest" + "go.temporal.io/server/common" + "go.temporal.io/server/common/clock" + "go.temporal.io/server/common/testing/mockapi/workflowservicemock/v1" + "go.uber.org/mock/gomock" + "google.golang.org/grpc" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// startFaults counts the upcoming StartWorkflowExecution RPCs that fail. +type startFaults struct { + rejected int // the RPC fails without the start being accepted + lostReply int // the start is accepted but the response is lost +} + +// fakeServices is the world outside the scheduler: a frontend that accepts +// starts into an external ledger, a history service that accepts terminate and +// cancel requests, and the counters and fault injection the invariants read. +type fakeServices struct { + Frontend *workflowservicemock.MockWorkflowServiceClient + History *historyservicemock.MockHistoryServiceClient + + // external records the starts the fake frontend accepted, outside the + // scheduler's transactions. + external *externalLedger + // startCalls is non-nil under WithControlledStarts, where a test decides + // each start RPC's outcome. + startCalls chan *StartCall + + // rpcMu guards the fields below: InvokerExecuteTask issues starts from + // concurrent goroutines. + rpcMu sync.Mutex + // actionRPCs counts every action RPC (StartWorkflowExecution, + // TerminateWorkflowExecution, RequestCancelWorkflowExecution), so a step can + // measure how many actions a single ExecuteTask took. + actionRPCs int + // startRequests records the RequestId of every StartWorkflowExecution RPC. + startRequests map[string]struct{} + // faults injects failures into uncontrolled start RPCs. + faults startFaults +} + +// newFakeServices wires happy-path stubs; tests can layer more specific +// expectations on Frontend and History. +func newFakeServices(t chasmtest.TB, ctrl *gomock.Controller, ts clock.TimeSource, controlledStarts bool) *fakeServices { + f := &fakeServices{ + Frontend: workflowservicemock.NewMockWorkflowServiceClient(ctrl), + History: historyservicemock.NewMockHistoryServiceClient(ctrl), + external: &externalLedger{starts: make(map[string]ExternalStart)}, + startRequests: make(map[string]struct{}), + } + if controlledStarts { + f.startCalls = make(chan *StartCall) + } + + f.Frontend.EXPECT().StartWorkflowExecution(gomock.Any(), gomock.Any()). + DoAndReturn(func(ctx context.Context, req *workflowservice.StartWorkflowExecutionRequest, _ ...grpc.CallOption) (*workflowservice.StartWorkflowExecutionResponse, error) { + f.rpcMu.Lock() + f.actionRPCs++ + f.startRequests[req.GetRequestId()] = struct{}{} + f.rpcMu.Unlock() + if f.startCalls != nil { + return f.controlledStart(ctx, t, req) + } + f.rpcMu.Lock() + rejected, lostReply := f.faults.rejected > 0, f.faults.rejected == 0 && f.faults.lostReply > 0 + if rejected { + f.faults.rejected-- + } else if lostReply { + f.faults.lostReply-- + } + f.rpcMu.Unlock() + if rejected { + return nil, serviceerror.NewUnavailable("start rejected by driver") + } + accepted, err := f.external.accept(req) + if err != nil { + return nil, err + } + if lostReply { + return nil, serviceerror.NewUnavailable("start accepted, response lost by driver") + } + return &workflowservice.StartWorkflowExecutionResponse{RunId: accepted.RunID}, nil + }).AnyTimes() + f.History.EXPECT().TerminateWorkflowExecution(gomock.Any(), gomock.Any()). + DoAndReturn(func(context.Context, *historyservice.TerminateWorkflowExecutionRequest, ...grpc.CallOption) (*historyservice.TerminateWorkflowExecutionResponse, error) { + f.countAction() + return &historyservice.TerminateWorkflowExecutionResponse{}, nil + }).AnyTimes() + f.History.EXPECT().RequestCancelWorkflowExecution(gomock.Any(), gomock.Any()). + DoAndReturn(func(context.Context, *historyservice.RequestCancelWorkflowExecutionRequest, ...grpc.CallOption) (*historyservice.RequestCancelWorkflowExecutionResponse, error) { + f.countAction() + return &historyservice.RequestCancelWorkflowExecutionResponse{}, nil + }).AnyTimes() + // The callbacks task (used for migrated running workflows) describes each + // running workflow; report it completed so watchers resolve deterministically. + f.History.EXPECT().DescribeWorkflowExecution(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, _ *historyservice.DescribeWorkflowExecutionRequest, _ ...grpc.CallOption) (*historyservice.DescribeWorkflowExecutionResponse, error) { + return &historyservice.DescribeWorkflowExecutionResponse{ + WorkflowExecutionInfo: &workflowpb.WorkflowExecutionInfo{ + Status: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, + CloseTime: timestamppb.New(ts.Now()), + }, + }, nil + }).AnyTimes() + return f +} + +// controlledStart hands the RPC to the test and waits for its Respond. +func (f *fakeServices) controlledStart(ctx context.Context, t chasmtest.TB, req *workflowservice.StartWorkflowExecutionRequest) (*workflowservice.StartWorkflowExecutionResponse, error) { + call := &StartCall{Request: common.CloneProto(req), t: t, result: make(chan startResponse, 1), ledger: f.external} + select { + case f.startCalls <- call: + case <-ctx.Done(): + return nil, ctx.Err() + } + select { + case result := <-call.result: + return result.response, result.err + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +func (f *fakeServices) countAction() { + f.rpcMu.Lock() + defer f.rpcMu.Unlock() + f.actionRPCs++ +} + +func (f *fakeServices) actionCount() int { + f.rpcMu.Lock() + defer f.rpcMu.Unlock() + return f.actionRPCs +} + +func (f *fakeServices) startRequestsCopy() map[string]struct{} { + f.rpcMu.Lock() + defer f.rpcMu.Unlock() + return maps.Clone(f.startRequests) +} + +// FailNextStarts makes the next n StartWorkflowExecution RPCs fail before the +// start is accepted, then lets later RPCs (the scheduler's retries) succeed. +// It does not apply under WithControlledStarts, where the test decides each +// RPC's outcome. +func (f *fakeServices) FailNextStarts(n int) { + f.rpcMu.Lock() + defer f.rpcMu.Unlock() + f.faults.rejected += n +} + +// LoseNextStartResponses makes the next n StartWorkflowExecution RPCs (after +// any FailNextStarts) accept the start but return an error, so the scheduler +// must retry with the same request ID and rely on deduplication. +func (f *fakeServices) LoseNextStartResponses(n int) { + f.rpcMu.Lock() + defer f.rpcMu.Unlock() + f.faults.lostReply += n +} diff --git a/chasm/lib/scheduler/schedulertest/inflight_test.go b/chasm/lib/scheduler/schedulertest/inflight_test.go new file mode 100644 index 00000000000..3e180941121 --- /dev/null +++ b/chasm/lib/scheduler/schedulertest/inflight_test.go @@ -0,0 +1,160 @@ +package schedulertest + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + enumspb "go.temporal.io/api/enums/v1" + "go.temporal.io/api/serviceerror" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/scheduler" +) + +func TestStartResultSurvivesBufferReordering(t *testing.T) { + t.Parallel() + schedule := DefaultSchedule() + schedule.State.Paused = true + schedule.Policies.OverlapPolicy = enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL + d := NewDriver(t, WithSchedule(schedule), WithControlledStarts()) + require.NoError(t, d.TriggerImmediately()) + require.NoError(t, d.TriggerImmediately()) + require.True(t, d.BeginSideEffect()) + first, second := d.NextStart(), d.NextStart() + acceptedFirst, err := first.Accept() + require.NoError(t, err) + acceptedSecond, err := second.Accept() + require.NoError(t, err) + _, err = d.Engine.UpdateComponent(chasm.NewEngineContext(context.Background(), d.Engine), d.Ref, + func(ctx chasm.MutableContext, c chasm.Component) error { + starts := c.(*scheduler.Scheduler).Invoker.Get(ctx).BufferedStarts + starts[0], starts[1] = starts[1], starts[0] + return nil + }) + require.NoError(t, err) + second.Respond(acceptedSecond, nil) + first.Respond(acceptedFirst, nil) + d.FinishSideEffect() + snap := d.Snapshot() + require.Empty(t, CheckInvariants(nil, snap)) + require.Equal(t, int64(2), snap.ActionCount) + require.Equal(t, acceptedFirst, snap.RecordedStarts[first.Request.GetRequestId()]) + require.Equal(t, acceptedSecond, snap.RecordedStarts[second.Request.GetRequestId()]) +} + +// selectedStarts counts starts chosen for execution whose workflow has not completed. +func selectedStarts(d *Driver) (selected int) { + d.ReadScheduler(func(s *scheduler.Scheduler, ctx chasm.Context) { + for _, start := range s.Invoker.Get(ctx).BufferedStarts { + if start.GetAttempt() > 0 && start.GetCompleted() == nil { + selected++ + } + } + }) + return selected +} + +func TestSelectedStartReservesOverlapSlot(t *testing.T) { + t.Parallel() + for _, policy := range []enumspb.ScheduleOverlapPolicy{ + enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ALL, + enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ONE, + } { + t.Run(policy.String(), func(t *testing.T) { + t.Parallel() + schedule := DefaultSchedule() + schedule.State.Paused = true + schedule.Policies.OverlapPolicy = policy + d := NewDriver(t, WithSchedule(schedule), WithControlledStarts()) + require.NoError(t, d.TriggerImmediately()) + require.True(t, d.BeginSideEffect()) + call := d.NextStart() + accepted, err := call.Accept() + require.NoError(t, err) + require.NoError(t, d.TriggerImmediately()) + require.Equal(t, 1, selectedStarts(d), "second start selected before the first acknowledgement") + call.Respond(accepted, nil) + d.FinishSideEffect() + require.Equal(t, 1, selectedStarts(d)) + + require.NoError(t, d.DeliverCompletion(call.Request.GetRequestId())) + require.True(t, d.StepPure()) + require.Equal(t, 1, selectedStarts(d), "second start not selected after the first completed") + require.Empty(t, CheckInvariants(nil, d.Snapshot())) + }) + } +} + +func TestFailedSelectedStartReleasesOverlapSlot(t *testing.T) { + t.Parallel() + schedule := DefaultSchedule() + schedule.State.Paused = true + schedule.Policies.OverlapPolicy = enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ALL + d := NewDriver(t, WithSchedule(schedule), WithControlledStarts()) + require.NoError(t, d.TriggerImmediately()) + require.True(t, d.BeginSideEffect()) + call := d.NextStart() + require.NoError(t, d.TriggerImmediately()) + require.Equal(t, 1, selectedStarts(d)) + call.Respond(ExternalStart{}, serviceerror.NewInvalidArgument("rejected")) + d.FinishSideEffect() + + require.True(t, d.StepPure()) + require.Equal(t, 1, selectedStarts(d), "second start not selected after the first was dropped") + require.Empty(t, CheckInvariants(nil, d.Snapshot())) +} + +func TestSelectedStartReservationStillTerminatesOther(t *testing.T) { + t.Parallel() + schedule := DefaultSchedule() + schedule.State.Paused = true + schedule.Policies.OverlapPolicy = enumspb.SCHEDULE_OVERLAP_POLICY_TERMINATE_OTHER + d := NewDriver(t, WithSchedule(schedule), WithControlledStarts()) + require.NoError(t, d.TriggerImmediately()) + require.True(t, d.BeginSideEffect()) + call := d.NextStart() + accepted, err := call.Accept() + require.NoError(t, err) + require.NoError(t, d.TriggerImmediately()) + call.Respond(accepted, nil) + d.FinishSideEffect() + require.True(t, d.StepPure()) + terminates := 0 + d.ReadScheduler(func(s *scheduler.Scheduler, ctx chasm.Context) { + terminates = len(s.Invoker.Get(ctx).TerminateWorkflows) + }) + require.Equal(t, 1, terminates, "deferred start never terminated the running workflow") +} + +// Migration is refused while a start is ready to execute, since V1 cannot resume +// it, and succeeds once the invoker has started it. The migration-untransferable +// invariant stays as a defence in case the refusal regresses. +func TestMigrationRefusedWhileExecutingObligations(t *testing.T) { + t.Parallel() + schedule := DefaultSchedule() + schedule.State.Paused = true + d := NewDriver(t, WithSchedule(schedule)) + require.NoError(t, d.TriggerImmediately()) + require.ErrorIs(t, d.MigrateToWorkflow(), scheduler.ErrMigrationActionsPending) + snap := d.Snapshot() + require.False(t, snap.Migrating) + require.Equal(t, 1, snap.UntransferableActions) + require.Empty(t, CheckInvariants(nil, snap)) + + for range 20 { + if d.Snapshot().UntransferableActions == 0 { + break + } + require.True(t, d.Step()) + } + require.Equal(t, 0, d.Snapshot().UntransferableActions) + require.NoError(t, d.MigrateToWorkflow()) + require.True(t, d.Snapshot().Migrating) + require.Empty(t, CheckInvariants(nil, d.Snapshot())) +} + +func TestMigrationUntransferableInvariant(t *testing.T) { + t.Parallel() + violations := CheckInvariants(nil, Snapshot{Migrating: true, UntransferableActions: 1}) + require.Contains(t, violations, Violation{Name: "migration-untransferable", Message: "1 execution/retry/cancel/terminate obligations lost native task ownership"}) +} diff --git a/chasm/lib/scheduler/schedulertest/invariants.go b/chasm/lib/scheduler/schedulertest/invariants.go new file mode 100644 index 00000000000..16c089415a7 --- /dev/null +++ b/chasm/lib/scheduler/schedulertest/invariants.go @@ -0,0 +1,209 @@ +package schedulertest + +import ( + "fmt" + "maps" + "slices" + "strings" +) + +// Violation describes a single broken invariant. +type Violation struct { + Name string + Message string +} + +func (v Violation) String() string { + return fmt.Sprintf("[%s] %s", v.Name, v.Message) +} + +// violations accumulates the broken invariants found by a check. +type violations []Violation + +func (vs *violations) add(name, format string, args ...any) { + *vs = append(*vs, Violation{Name: name, Message: fmt.Sprintf(format, args...)}) +} + +// CheckInvariants evaluates the scheduler invariants against a settled state +// (cur). When prev is non-nil it also checks cross-step invariants (monotonicity, +// closed-is-terminal). It returns all violations found; an empty slice means the +// state is healthy. +// +// The invariants encode the legitimate-quiescence rules from Scheduler.isHeldOpen +// and Scheduler.getIdleExpiration: a non-closed schedule may have no pending task +// only when it is held open (paused or draining a backfill). Any other taskless, +// non-closed state is the "stuck" bug class. +func CheckInvariants(prev *Snapshot, cur Snapshot) []Violation { + var vs violations + + // (1) No-stuck (primary): a live schedule must always have a way to make + // progress. The only taskless, non-closed state allowed is "held open" + // (paused or pending backfill). When idle is armed there is a pending idle + // task, so HasPendingTask covers that case. + // + // The engine also asserts a stuck check on every commit (StateInvariant), + // but this rule is stronger in one way and is kept alongside it: + // HasPendingTask counts only side-effect tasks whose handler still considers + // them valid and ignores the framework visibility task, whereas the engine + // check counts every logical task, including the visibility task that a test + // engine never dispatches. The engine check is stronger in the other + // direction (no held-open exemption), which is why both run. + if !cur.Closed && !cur.HasPendingTask && !cur.IsHeldOpen { + vs.add("no-stuck", + "scheduler has no pending task, is not closed, and is not held open "+ + "(paused=%v backfillers=%d) at now=%s — stuck", + cur.Paused, cur.BackfillerCount, cur.Now.Format("2006-01-02T15:04:05")) + } + + // (2) Idle correctness: a held-open schedule must never have an idle-close + // armed; idle close would race the customer's intent to resume/drain. + if cur.IsHeldOpen && !cur.IdleCloseTime.IsZero() { + vs.add("idle-not-while-held-open", + "scheduler is held open (paused=%v backfillers=%d) but has idle close armed at %s", + cur.Paused, cur.BackfillerCount, cur.IdleCloseTime.Format("2006-01-02T15:04:05")) + } + + // (2a) Buffer bound: the total buffered starts must never exceed MaxBufferSize. + // The generator sizes each fill against this cap (remaining = MaxBufferSize - + // len(buffered)); exceeding it means the limit was not enforced and CHASM state + // can grow without bound (oversized state, persistence failures, OOM). + if cur.MaxBufferSize > 0 && cur.BufferedStartsCount > cur.MaxBufferSize { + vs.add("buffer-bound", + "buffered starts = %d, exceeding MaxBufferSize=%d", + cur.BufferedStartsCount, cur.MaxBufferSize) + } + + // (2b) Action budget: a single InvokerExecuteTask must not take more actions + // (start + terminate + cancel RPCs) than MaxActionsPerExecution. Exceeding it + // means the per-execution rate protection was bypassed — e.g. the action budget + // reset across the terminate/cancel/start phases, letting one execution take up + // to a multiple of the configured maximum. + if cur.MaxActionsPerExecution > 0 && cur.MaxActionsObserved > cur.MaxActionsPerExecution { + vs.add("action-budget", + "a single ExecuteTask took %d actions, exceeding MaxActionsPerExecution=%d", + cur.MaxActionsObserved, cur.MaxActionsPerExecution) + } + + vs = append(vs, checkLedger(prev, cur)...) + vs = append(vs, checkStartAccounting(cur)...) + + if prev != nil { + // (3) Closed is terminal: a closed schedule must never reopen. + if prev.Closed && !cur.Closed { + vs.add("closed-terminal", "scheduler reopened after being closed") + } + + // (4) High-water-mark monotonicity: generator/invoker HWMs never regress. + if cur.GeneratorLPT.Before(prev.GeneratorLPT) { + vs.add("hwm-monotonic-generator", + "generator high-water mark went backward: %s -> %s", + prev.GeneratorLPT.Format("2006-01-02T15:04:05"), + cur.GeneratorLPT.Format("2006-01-02T15:04:05")) + } + if cur.InvokerLPT.Before(prev.InvokerLPT) { + vs.add("hwm-monotonic-invoker", + "invoker high-water mark went backward: %s -> %s", + prev.InvokerLPT.Format("2006-01-02T15:04:05"), + cur.InvokerLPT.Format("2006-01-02T15:04:05")) + } + } + + return vs +} + +// fataler is the subset of testing.TB the invariant assertion helper needs. +type fataler interface { + Helper() + Fatalf(format string, args ...any) +} + +// CheckInvariantsHook returns an AfterStep hook that captures a snapshot after +// every step and fails the test, reporting every invariant violation found. It threads the +// previous snapshot so cross-step invariants are checked. This is the normal way +// invariant checking is wired into a Driver: +// +// d := NewDriver(t) +// d.AfterStep = CheckInvariantsHook(t) +func CheckInvariantsHook(t fataler) func(d *Driver) { + var prev *Snapshot + return func(d *Driver) { + t.Helper() + cur := d.Snapshot() + if vs := CheckInvariants(prev, cur); len(vs) > 0 { + msgs := make([]string, len(vs)) + for i, v := range vs { + msgs[i] = v.String() + } + t.Fatalf("scheduler invariant violation at step (now=%s): %s", + cur.Now.Format("2006-01-02T15:04:05"), strings.Join(msgs, "; ")) + } + snap := cur + prev = &snap + } +} + +// checkLedger checks that every start the scheduler buffered is either sent to +// StartWorkflowExecution or discarded on purpose, and never lost to a close. +func checkLedger(prev *Snapshot, cur Snapshot) []Violation { + var vs violations + + if cur.Migrating && cur.UntransferableActions > 0 { + vs.add("migration-untransferable", "%d execution/retry/cancel/terminate obligations lost native task ownership", cur.UntransferableActions) + } + if cur.Migrating && cur.Closed && len(cur.PendingStarts) > 0 { + vs.add("migration-transfer-unverified", "closed source retains %d obligations without a verified destination", len(cur.PendingStarts)) + } + + // Terminal half: a schedule never closes while a start it buffered is + // still waiting for InvokerExecuteTask (temporal#13016). + if cur.Closed && !cur.Migrating && len(cur.PendingStarts) > 0 { + vs.add("ledger-lost-start", + "scheduler closed with %d buffered start(s) never sent to StartWorkflowExecution: %v", + len(cur.PendingStarts), slices.Sorted(maps.Keys(cur.PendingStarts))) + } + if prev == nil { + return vs + } + + // Conservation half: every start that leaves the buffer without a RunId was + // either sent to StartWorkflowExecution or discarded on purpose (overlap + // skip, missed catchup, paused/limited drop). Starts that are buffered and + // discarded within one transaction never appear pending, so the discard + // counters can only exceed the disappearances, never trail. A start may also + // be replaced by a CANCEL_OTHER or TERMINATE_OTHER start (see + // Snapshot.ReplacingStarts). + var silent []string + for id := range prev.PendingStarts { + _, still := cur.PendingStarts[id] + _, sent := cur.StartRequested[id] + if !still && !sent { + silent = append(silent, id) + } + } + discarded := (cur.OverlapSkipped - prev.OverlapSkipped) + (cur.DroppedStarts - prev.DroppedStarts) + + int64(prev.ReplacingStarts+cur.ReplacingSeen-prev.ReplacingSeen) + if int64(len(silent)) > discarded { + slices.Sort(silent) + vs.add("ledger-silent-drop", + "%d buffered start(s) left the buffer without a start RPC, but only %d discard(s) were recorded: %v", + len(silent), discarded, silent) + } + return vs +} + +func checkStartAccounting(cur Snapshot) []Violation { + var vs violations + for id, accepted := range cur.AcceptedStarts { + recorded, ok := cur.RecordedStarts[id] + if ok && recorded != accepted { + vs.add("start-identity", "request %s recorded %+v, accepted %+v", id, recorded, accepted) + } + if _, pending := cur.PendingStarts[id]; !ok && !pending { + vs.add("accepted-start-lost", "accepted request %s has neither a pending obligation nor a recorded result", id) + } + } + if want := cur.InitialActionCount + int64(len(cur.RecordedStarts)); cur.ActionCount != want { + vs.add("start-accounting", "ActionCount=%d, observed recorded starts plus initial count=%d", cur.ActionCount, want) + } + return vs +} diff --git a/chasm/lib/scheduler/schedulertest/invariants_test.go b/chasm/lib/scheduler/schedulertest/invariants_test.go new file mode 100644 index 00000000000..7011925de5e --- /dev/null +++ b/chasm/lib/scheduler/schedulertest/invariants_test.go @@ -0,0 +1,141 @@ +package schedulertest_test + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + schedulepb "go.temporal.io/api/schedule/v1" + "go.temporal.io/server/chasm/lib/scheduler/schedulertest" + "google.golang.org/protobuf/types/known/durationpb" +) + +// TestInvariants_HoldAcrossScenarios runs a variety of schedules forward with +// the invariant hook wired in and asserts no invariant fires. Each scenario +// exercises a different legitimate-quiescence path (running, paused, exhausted, +// end-time passed). +func TestInvariants_HoldAcrossScenarios(t *testing.T) { + cases := []struct { + name string + schedule func() *schedulepb.Schedule + maxSteps int + }{ + { + name: "running interval", + schedule: schedulertest.DefaultSchedule, + maxSteps: 30, + }, + { + name: "paused interval (held open, keeps ticking)", + schedule: func() *schedulepb.Schedule { + s := schedulertest.DefaultSchedule() + s.State.Paused = true + return s + }, + maxSteps: 15, + }, + { + name: "limited actions exhausted (idles then closes)", + schedule: func() *schedulepb.Schedule { + s := schedulertest.DefaultSchedule() + s.State.LimitedActions = true + s.State.RemainingActions = 0 + return s + }, + maxSteps: 10, + }, + { + name: "limited actions, a few then idle", + schedule: func() *schedulepb.Schedule { + s := schedulertest.DefaultSchedule() + s.State.LimitedActions = true + s.State.RemainingActions = 3 + return s + }, + maxSteps: 30, + }, + { + name: "short interval, long catchup", + schedule: func() *schedulepb.Schedule { + s := schedulertest.DefaultSchedule() + s.Spec.Interval[0].Interval = durationpb.New(10 * time.Second) + return s + }, + maxSteps: 40, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + d := schedulertest.NewDriver(t, schedulertest.WithSchedule(tc.schedule())) + d.AfterStep = schedulertest.CheckInvariantsHook(t) + // The hook fails the test on any violation; we just drive it forward. + d.RunToQuiescence(tc.maxSteps) + }) + } +} + +// TestCheckInvariants_DetectsStuck is a direct unit test of the invariant +// predicate: a non-closed schedule with no pending task that is not held open is +// flagged as stuck. +func TestCheckInvariants_DetectsStuck(t *testing.T) { + stuck := schedulertest.Snapshot{ + Now: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC), + HasPendingTask: false, + Closed: false, + IsHeldOpen: false, + } + violations := schedulertest.CheckInvariants(nil, stuck) + require.NotEmpty(t, violations, "stuck state must be flagged") + require.Equal(t, "no-stuck", violations[0].Name) +} + +// TestCheckInvariants_AllowsHeldOpenWithoutTask confirms a paused schedule with +// no pending task is NOT flagged (legitimate hold-open). +func TestCheckInvariants_AllowsHeldOpenWithoutTask(t *testing.T) { + heldOpen := schedulertest.Snapshot{ + Now: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC), + HasPendingTask: false, + Closed: false, + Paused: true, + IsHeldOpen: true, + } + require.Empty(t, schedulertest.CheckInvariants(nil, heldOpen)) +} + +// TestCheckInvariants_DetectsHwmRegression confirms the high-water-mark +// monotonicity invariant fires when the generator HWM moves backward. +func TestCheckInvariants_DetectsHwmRegression(t *testing.T) { + t0 := time.Date(2020, 1, 1, 0, 5, 0, 0, time.UTC) + prev := schedulertest.Snapshot{HasPendingTask: true, GeneratorLPT: t0} + cur := schedulertest.Snapshot{HasPendingTask: true, GeneratorLPT: t0.Add(-time.Minute)} + violations := schedulertest.CheckInvariants(&prev, cur) + require.NotEmpty(t, violations) + require.Equal(t, "hwm-monotonic-generator", violations[0].Name) +} + +// TestCheckInvariants_DetectsBufferOverrun confirms the buffer-bound invariant +// fires when buffered starts exceed MaxBufferSize, and that being exactly at the +// cap is allowed. +func TestCheckInvariants_DetectsBufferOverrun(t *testing.T) { + over := schedulertest.Snapshot{HasPendingTask: true, MaxBufferSize: 1000, BufferedStartsCount: 1001} + violations := schedulertest.CheckInvariants(nil, over) + require.NotEmpty(t, violations, "buffer overrun must be flagged") + require.Equal(t, "buffer-bound", violations[0].Name) + + atCap := schedulertest.Snapshot{HasPendingTask: true, MaxBufferSize: 1000, BufferedStartsCount: 1000} + require.Empty(t, schedulertest.CheckInvariants(nil, atCap), "exactly at the cap is allowed") +} + +// TestCheckInvariants_DetectsBudgetOverrun confirms the action-budget invariant +// fires when a single ExecuteTask takes more actions than MaxActionsPerExecution +// (the double-spend bug class), and that being exactly at the cap is allowed. +func TestCheckInvariants_DetectsBudgetOverrun(t *testing.T) { + over := schedulertest.Snapshot{HasPendingTask: true, MaxActionsPerExecution: 5, MaxActionsObserved: 6} + violations := schedulertest.CheckInvariants(nil, over) + require.NotEmpty(t, violations, "budget overrun must be flagged") + require.Equal(t, "action-budget", violations[0].Name) + + atCap := schedulertest.Snapshot{HasPendingTask: true, MaxActionsPerExecution: 5, MaxActionsObserved: 5} + require.Empty(t, schedulertest.CheckInvariants(nil, atCap), "exactly at the cap is allowed") +} diff --git a/chasm/lib/scheduler/schedulertest/ledger_test.go b/chasm/lib/scheduler/schedulertest/ledger_test.go new file mode 100644 index 00000000000..04b7465c848 --- /dev/null +++ b/chasm/lib/scheduler/schedulertest/ledger_test.go @@ -0,0 +1,211 @@ +package schedulertest_test + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + enumspb "go.temporal.io/api/enums/v1" + schedulepb "go.temporal.io/api/schedule/v1" + "go.temporal.io/server/chasm/lib/scheduler" + "go.temporal.io/server/chasm/lib/scheduler/schedulertest" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// configWith returns DefaultConfig with Tweakables adjusted by fn. +func configWith(fn func(*scheduler.Tweakables)) *scheduler.Config { + cfg := schedulertest.DefaultConfig() + tweakables := scheduler.DefaultTweakables + fn(&tweakables) + cfg.Tweakables = func(string) scheduler.Tweakables { return tweakables } + return cfg +} + +// oneShotSchedule fires exactly once, at start+offset. +func oneShotSchedule(start time.Time, offset time.Duration) *schedulepb.Schedule { + s := schedulertest.DefaultSchedule() + s.Spec.Interval[0].Interval = durationpb.New(time.Hour) + s.Spec.Interval[0].Phase = durationpb.New(offset % time.Hour) + s.Spec.StartTime = timestamppb.New(start.Add(offset)) + s.Spec.EndTime = timestamppb.New(start.Add(offset + time.Minute)) + return s +} + +// collectViolations records every invariant violation instead of failing, so a +// test can assert that a known bug is detected. +func collectViolations(out *[]schedulertest.Violation) func(*schedulertest.Driver) { + var prev *schedulertest.Snapshot + return func(d *schedulertest.Driver) { + cur := d.Snapshot() + *out = append(*out, schedulertest.CheckInvariants(prev, cur)...) + prev = &cur + } +} + +func violationNames(vs []schedulertest.Violation) []string { + names := make([]string, 0, len(vs)) + for _, v := range vs { + names = append(names, v.Name) + } + return names +} + +// The final action of a schedule is buffered by the same generator tick that +// arms the idle task. If InvokerExecuteTask hasn't run by the time the idle task +// fires, the schedule must stay open. Pre-#13016 the idle deadline ignored the +// pending start and was already in the past. +func TestLedger_FinalStartPendingAtIdle(t *testing.T) { + const idle = time.Hour + start := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) + + cases := []struct { + name string + fixOn bool + offset time.Duration + wantBad []string + }{ + {"fix on, final gap > IdleTime", true, 3 * idle, nil}, + {"fix on, final gap < IdleTime", true, idle / 2, nil}, + {"fix off, final gap > IdleTime", false, 3 * idle, []string{"ledger-lost-start"}}, + {"fix off, final gap < IdleTime", false, idle / 2, nil}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var violations []schedulertest.Violation + d := schedulertest.NewDriver(t, + schedulertest.WithStartTime(start), + schedulertest.WithSchedule(oneShotSchedule(start, tc.offset)), + schedulertest.WithConfig(configWith(func(tw *scheduler.Tweakables) { + tw.IdleTime = idle + tw.EnablePendingStartIdleFix = tc.fixOn + })), + schedulertest.WithMaxSideEffectLag(idle/2), + schedulertest.WithAfterStep(collectViolations(&violations)), + ) + + // Fire timers while InvokerExecuteTask lags, then drain. + for d.StepPure() { + } + _, quiescent := d.RunToQuiescence(50) + require.True(t, quiescent) + + require.Equal(t, tc.wantBad, uniq(violationNames(violations))) + snap := d.Snapshot() + require.True(t, snap.Closed) + if tc.wantBad == nil { + require.Len(t, snap.StartRequested, 1, "the one action must start") + } + }) + } +} + +// A buffered start that vanishes with no RPC and no recorded discard is a +// silent drop; one with a recorded discard is not. +func TestCheckInvariants_Ledger(t *testing.T) { + pending := func(ids ...string) map[string]struct{} { + m := make(map[string]struct{}, len(ids)) + for _, id := range ids { + m[id] = struct{}{} + } + return m + } + + cases := []struct { + name string + prev, cur schedulertest.Snapshot + want []string + }{ + { + name: "started", + prev: schedulertest.Snapshot{HasPendingTask: true, PendingStarts: pending("a")}, + cur: schedulertest.Snapshot{HasPendingTask: true, StartRequested: pending("a")}, + }, + { + name: "overlap skipped", + prev: schedulertest.Snapshot{HasPendingTask: true, PendingStarts: pending("a")}, + cur: schedulertest.Snapshot{HasPendingTask: true, OverlapSkipped: 1}, + }, + { + name: "dropped", + prev: schedulertest.Snapshot{HasPendingTask: true, PendingStarts: pending("a", "b"), DroppedStarts: 3}, + cur: schedulertest.Snapshot{HasPendingTask: true, DroppedStarts: 4, StartRequested: pending("b")}, + }, + { + name: "replaced by cancel-other start", + prev: schedulertest.Snapshot{HasPendingTask: true, PendingStarts: pending("a", "b"), ReplacingStarts: 1}, + cur: schedulertest.Snapshot{HasPendingTask: true, StartRequested: pending("b")}, + }, + { + name: "silent drop", + prev: schedulertest.Snapshot{HasPendingTask: true, PendingStarts: pending("a", "b")}, + cur: schedulertest.Snapshot{HasPendingTask: true, OverlapSkipped: 1}, + want: []string{"ledger-silent-drop"}, + }, + { + name: "closed with pending start", + prev: schedulertest.Snapshot{HasPendingTask: true, PendingStarts: pending("a")}, + cur: schedulertest.Snapshot{Closed: true, PendingStarts: pending("a")}, + want: []string{"ledger-lost-start"}, + }, + { + name: "migration needs destination evidence", + want: []string{"migration-transfer-unverified"}, + prev: schedulertest.Snapshot{HasPendingTask: true, PendingStarts: pending("a")}, + cur: schedulertest.Snapshot{Closed: true, Migrating: true, PendingStarts: pending("a")}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := schedulertest.CheckInvariants(&tc.prev, tc.cur) + require.Equal(t, tc.want, uniq(violationNames(got))) + }) + } +} + +func uniq(names []string) []string { + var out []string + seen := map[string]bool{} + for _, n := range names { + if !seen[n] { + seen[n] = true + out = append(out, n) + } + } + return out +} + +// A start deferred behind a running workflow is only re-enabled by a workflow +// completion. If that completion arrives while the schedule is paused, +// ProcessBuffer drops the first start (paused) and re-defers the rest with +// nothing running, so no completion will ever re-enable them. Unpause doesn't +// re-process the buffer either; the starts (including a manual trigger, which +// pause allows) sit until the idle task closes the schedule. +func TestLedger_DeferredStartsStrandedByPausedCompletion(t *testing.T) { + + const idle = time.Hour + start := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) + s := schedulertest.DefaultSchedule() + s.Policies.OverlapPolicy = enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ALL + s.Spec.EndTime = timestamppb.New(start.Add(2*time.Minute + time.Second)) + + var violations []schedulertest.Violation + d := schedulertest.NewDriver(t, + schedulertest.WithStartTime(start), + schedulertest.WithSchedule(s), + schedulertest.WithConfig(configWith(func(tw *scheduler.Tweakables) { tw.IdleTime = idle })), + schedulertest.WithAfterStep(collectViolations(&violations)), + ) + + require.True(t, d.Step()) // 00:01 starts, keeps running + require.True(t, d.Step()) // 00:02 final fire, deferred behind it + require.NoError(t, d.Pause()) + require.NoError(t, d.TriggerImmediately()) // manual, deferred behind it + require.True(t, d.CompleteWorkflow()) + require.NoError(t, d.Unpause()) + _, quiescent := d.RunToQuiescence(50) + require.True(t, quiescent) + + require.Empty(t, violationNames(violations)) + require.Len(t, d.Snapshot().StartRequested, 2, "the manual trigger must start") +} diff --git a/chasm/lib/scheduler/schedulertest/library.go b/chasm/lib/scheduler/schedulertest/library.go new file mode 100644 index 00000000000..3a1432a2adf --- /dev/null +++ b/chasm/lib/scheduler/schedulertest/library.go @@ -0,0 +1,219 @@ +// Package schedulertest provides a lifecycle-driving harness for the CHASM +// scheduler. Unlike the per-handler unit tests in package scheduler_test, this +// package builds a real component on top of [chasmtest.Engine] and steps it +// forward through every task it schedules for itself, so that invariants can be +// asserted across sequences of transitions (e.g. "the scheduler never gets +// stuck without a task to drive the next state"). +// +// It deliberately lives in a non-test package so that property tests +// (pgregory.net/rapid), the V1<->V2 migration round-trip test and the unit tests +// in package scheduler_test can all import it. It is the single home of the +// scheduler fixtures (default schedule and config, spec processor, library +// wiring) and of [SchedulerEngine], the typed wrapper around a chasmtest engine +// that helper_test.go and [Driver] are both built on. +package schedulertest + +import ( + "time" + + commonpb "go.temporal.io/api/common/v1" + schedulepb "go.temporal.io/api/schedule/v1" + workflowpb "go.temporal.io/api/workflow/v1" + "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/server/chasm/lib/scheduler" + "go.temporal.io/server/common/backoff" + "go.temporal.io/server/common/clock" + "go.temporal.io/server/common/dynamicconfig" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/resource" + "go.temporal.io/server/common/searchattribute" + legacyscheduler "go.temporal.io/server/service/worker/scheduler" + "go.uber.org/mock/gomock" + "google.golang.org/protobuf/types/known/durationpb" +) + +const ( + Namespace = "ns" + NamespaceID = "ns-id" + ScheduleID = "sched-id" + + DefaultInterval = 1 * time.Minute + DefaultCatchupWindow = 5 * time.Minute +) + +// DefaultSchedule returns a schedule with a single 1-minute interval and a +// 5-minute catchup window, matching the package's other defaults. +func DefaultSchedule() *schedulepb.Schedule { + return &schedulepb.Schedule{ + Spec: &schedulepb.ScheduleSpec{ + Interval: []*schedulepb.IntervalSpec{ + { + Interval: durationpb.New(DefaultInterval), + Phase: durationpb.New(0), + }, + }, + }, + Action: &schedulepb.ScheduleAction{ + Action: &schedulepb.ScheduleAction_StartWorkflow{ + StartWorkflow: &workflowpb.NewWorkflowExecutionInfo{ + WorkflowId: "scheduled-wf", + WorkflowType: &commonpb.WorkflowType{Name: "scheduled-wf-type"}, + }, + }, + }, + Policies: &schedulepb.SchedulePolicies{ + CatchupWindow: durationpb.New(DefaultCatchupWindow), + }, + State: &schedulepb.ScheduleState{ + Paused: false, + LimitedActions: false, + RemainingActions: 0, + }, + } +} + +// DefaultConfig returns a scheduler config wired with the package defaults. +func DefaultConfig() *scheduler.Config { + return &scheduler.Config{ + Tweakables: func(_ string) scheduler.Tweakables { + return scheduler.DefaultTweakables + }, + ServiceCallTimeout: func() time.Duration { + return 5 * time.Second + }, + EncodeInternalTokenWithEnvelope: func(string) bool { + return true + }, + RetryPolicy: func() backoff.RetryPolicy { return deterministicRetryPolicy{} }, + } +} + +// deterministicRetryPolicy backs off exponentially from one second to a minute +// like the production policy but without its random jitter, so that a rapid +// run replays identically while it shrinks a failure. +type deterministicRetryPolicy struct{} + +func (deterministicRetryPolicy) ComputeNextDelay(_ time.Duration, numAttempts int, _ error) time.Duration { + return min(time.Second< 0 { + s.Spec.Jitter = durationpb.New(time.Duration(jitterSec) * time.Second) + } + + if catchupSec := rapid.SampledFrom([]int64{60, 300, 3600}).Draw(rt, "catchupSec"); catchupSec > 0 { + s.Policies.CatchupWindow = durationpb.New(time.Duration(catchupSec) * time.Second) + } + + s.Policies.OverlapPolicy = drawOverlapPolicy(rt, "overlapPolicy", policies) + + switch rapid.SampledFrom([]string{"forever", "endTime", "oneShot"}).Draw(rt, "shape") { + case "endTime": + // Last fire lands a few intervals in, so the final gap is the interval. + n := rapid.IntRange(1, 4).Draw(rt, "firesBeforeEnd") + s.Spec.EndTime = timestamppb.New(start.Add(time.Duration(n)*interval + time.Second)) + case "oneShot": + // A single fire, before, at, or after the idle window since creation. + offset := rapid.SampledFrom([]time.Duration{ + idle / 2, idle - time.Minute, idle, idle + time.Minute, 3 * idle, + }).Draw(rt, "oneShotOffset") + s.Spec.Interval[0].Interval = durationpb.New(time.Hour) + s.Spec.Interval[0].Phase = durationpb.New(offset % time.Hour) + s.Spec.StartTime = timestamppb.New(start.Add(offset)) + s.Spec.EndTime = timestamppb.New(start.Add(offset + time.Minute)) + default: + } + + if rapid.Bool().Draw(rt, "paused") { + s.State.Paused = true + } + if rapid.Bool().Draw(rt, "limited") { + s.State.LimitedActions = true + s.State.RemainingActions = int64(rapid.IntRange(0, 5).Draw(rt, "remainingActions")) + } + + return s +} + +func checkPatchError(t *rapid.T, err error) { + t.Helper() + if err != nil && !errors.Is(err, scheduler.ErrClosed) { + t.Fatalf("unexpected patch failure: %v", err) + } +} diff --git a/chasm/lib/scheduler/schedulertest/snapshot.go b/chasm/lib/scheduler/schedulertest/snapshot.go new file mode 100644 index 00000000000..11a17d449d4 --- /dev/null +++ b/chasm/lib/scheduler/schedulertest/snapshot.go @@ -0,0 +1,172 @@ +package schedulertest + +import ( + "cmp" + "maps" + "time" + + enumspb "go.temporal.io/api/enums/v1" + schedulespb "go.temporal.io/server/api/schedule/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/scheduler" + "go.temporal.io/server/common/metrics" +) + +// Snapshot captures the observable scheduler state at a settled point (after a +// step's CloseTransaction). It is the input to invariant checks. +type Snapshot struct { + UntransferableActions int + AcceptedStarts map[string]ExternalStart + RecordedStarts map[string]ExternalStart + ActionCount int64 + InitialActionCount int64 + + // Now is the engine clock at capture time. + Now time.Time + // HasPendingTask is true if any task is scheduled to fire. + HasPendingTask bool + + Closed bool + IsSentinel bool + Paused bool + // BackfillerCount is the number of live backfiller components. + BackfillerCount int + // IsHeldOpen mirrors Scheduler.isHeldOpen: the schedule must stay open + // regardless of having no work (paused or pending backfill, non-sentinel). + IsHeldOpen bool + // IdleCloseTime is the armed idle-close deadline, or zero if no idle task is armed. + IdleCloseTime time.Time + + // GeneratorLPT / InvokerLPT are the generator and invoker high-water marks. + GeneratorLPT time.Time + InvokerLPT time.Time + + // BufferedStartsCount is len(invoker.GetBufferedStarts()) — the total buffered + // starts (pending + retained). MaxBufferSize is the configured cap; the + // buffer-bound invariant requires BufferedStartsCount <= MaxBufferSize. + BufferedStartsCount int + MaxBufferSize int + + // MaxActionsObserved is the high-water mark of action RPCs (start + terminate + + // cancel) taken by any single InvokerExecuteTask so far. MaxActionsPerExecution + // is the configured per-execution cap; the action-budget invariant requires + // MaxActionsObserved <= MaxActionsPerExecution. + MaxActionsObserved int + MaxActionsPerExecution int + + // Start ledger inputs. PendingStarts are buffered starts with no RunId yet. + // StartRequested holds every RequestId sent to StartWorkflowExecution. + // OverlapSkipped and DroppedStarts count the starts the scheduler discarded + // on purpose (Info.OverlapSkipped, and the ScheduleBufferedStartDropped + // metric for missed-catchup and paused/limited drops). + PendingStarts map[string]struct{} + StartRequested map[string]struct{} + // ReplacingStarts counts the pending starts that no start RPC has been attempted for (including deferred ones) whose overlap policy + // is CANCEL_OTHER or TERMINATE_OTHER. With nothing running, such a start + // replaces the one the buffer would otherwise have started + // (legacyscheduler.ProcessBuffer), discarding it without counting an overlap + // skip, so each can account for one silent disappearance. ReplacingSeen is + // the number of distinct such starts ever seen in the buffer in any state, + // which covers a replacer that was buffered and started between two snapshots. + ReplacingStarts int + ReplacingSeen int + OverlapSkipped int64 + DroppedStarts int64 + // Migrating is true once the schedule hands off to a V1 workflow, which + // takes its buffered starts with it. + Migrating bool + // DeferredStarts are pending starts waiting on a workflow completion + // (Attempt == -1); RunningWorkflows counts started, uncompleted workflows. + DeferredStarts int + RunningWorkflows int +} + +// Snapshot reads current scheduler state into a Snapshot. +func (d *Driver) Snapshot() Snapshot { + d.T.Helper() + d.observeRecordedStarts() + tweakables := d.Config.Tweakables(Namespace) + snap := Snapshot{ + AcceptedStarts: d.external.snapshot(), RecordedStarts: maps.Clone(d.recorded), + InitialActionCount: d.initialActionCount, + Now: d.Now(), + HasPendingTask: d.HasPendingTask(), + MaxBufferSize: tweakables.MaxBufferSize, + MaxActionsObserved: d.maxActionsObserved, + MaxActionsPerExecution: tweakables.MaxActionsPerExecution, + PendingStarts: make(map[string]struct{}), + StartRequested: d.startRequestsCopy(), + DroppedStarts: d.droppedStarts(), + } + d.ReadScheduler(func(s *scheduler.Scheduler, ctx chasm.Context) { + snap.ActionCount = s.Info.GetActionCount() + snap.Closed = s.Closed + snap.IsSentinel = s.IsSentinel() + snap.Paused = s.Schedule.GetState().GetPaused() + snap.BackfillerCount = len(s.Backfillers) + snap.OverlapSkipped = s.Info.GetOverlapSkipped() + snap.Migrating = s.GetWorkflowMigration() != nil + // Mirror Scheduler.isHeldOpen() from public state. + snap.IsHeldOpen = !snap.IsSentinel && (snap.Paused || snap.BackfillerCount > 0) + if s.IdleCloseTime != nil { + snap.IdleCloseTime = s.IdleCloseTime.AsTime() + } + // Sentinels have no generator/invoker sub-components. + if !snap.IsSentinel { + if g := s.Generator.Get(ctx); g != nil && g.LastProcessedTime != nil { + snap.GeneratorLPT = g.LastProcessedTime.AsTime() + } + if inv := s.Invoker.Get(ctx); inv != nil { + snap.UntransferableActions = len(inv.GetCancelWorkflows()) + len(inv.GetTerminateWorkflows()) + snap.BufferedStartsCount = len(inv.GetBufferedStarts()) + snap.addBufferedStarts(inv.GetBufferedStarts(), s.Schedule.GetPolicies().GetOverlapPolicy(), d.replacingSeen) + snap.ReplacingSeen = len(d.replacingSeen) + if inv.LastProcessedTime != nil { + snap.InvokerLPT = inv.LastProcessedTime.AsTime() + } + } + } + }) + return snap +} + +// addBufferedStarts classifies buffered starts into pending (deferred or not) +// and running. scheduleOverlap resolves starts with an unspecified policy. +// replacingSeen accumulates the request IDs of replacing starts across snapshots. +func (snap *Snapshot) addBufferedStarts(starts []*schedulespb.BufferedStart, scheduleOverlap enumspb.ScheduleOverlapPolicy, replacingSeen map[string]struct{}) { + for _, start := range starts { + policy := cmp.Or(start.GetOverlapPolicy(), scheduleOverlap) + replacing := policy == enumspb.SCHEDULE_OVERLAP_POLICY_CANCEL_OTHER || + policy == enumspb.SCHEDULE_OVERLAP_POLICY_TERMINATE_OTHER + if replacing { + replacingSeen[start.GetRequestId()] = struct{}{} + } + switch { + case start.GetRunId() == "": + if replacing && start.GetAttempt() <= 0 { + snap.ReplacingStarts++ + } + if start.GetAttempt() > 0 { + snap.UntransferableActions++ + } + snap.PendingStarts[start.GetRequestId()] = struct{}{} + if start.GetAttempt() == -1 { + snap.DeferredStarts++ + } + case start.GetCompleted() == nil: + snap.RunningWorkflows++ + default: + } + } +} + +// droppedStarts sums the ScheduleBufferedStartDropped counter. +func (d *Driver) droppedStarts() int64 { + var total int64 + for _, rec := range d.metricsCapture.SnapshotMetric(metrics.ScheduleBufferedStartDropped.Name()) { + if v, ok := rec.Value.(int64); ok { + total += v + } + } + return total +} diff --git a/chasm/lib/scheduler/schedulertest/state_invariant.go b/chasm/lib/scheduler/schedulertest/state_invariant.go new file mode 100644 index 00000000000..ef16bb9e3ee --- /dev/null +++ b/chasm/lib/scheduler/schedulertest/state_invariant.go @@ -0,0 +1,288 @@ +package schedulertest + +import ( + "context" + "fmt" + "slices" + "strings" + "sync" + + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/chasmtest" + "go.temporal.io/server/chasm/lib/scheduler" + schedulerpb "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/metrics" + "google.golang.org/protobuf/proto" +) + +// This file implements the "stuckness" and idle-deadline invariants for +// Schedules/V2. [SchedulerEngine] installs them on every transition it commits, +// and the unit tests' testEnv asserts them on every CloseTransaction, so the +// whole suite and the lifecycle property check the same two predicates. +// +// Production has repeatedly produced schedules that are open, unpaused, and +// carry no pending work: they never re-arm and never close. Every confirmed +// instance violated exactly one predicate: +// +// !sched.Closed => the tree carries at least one live logical task. +// +// This is a liveness property, not a transition-validity property. The +// scheduler has no state enum to validate transitions against: +// Scheduler.LifecycleState is derived and two-valued (Completed iff Closed), +// and every sub-component hardcodes LifecycleStateRunning. In each known bug +// every transition taken was legal; the failure was that no transition +// happened at all. +// +// The stuck predicate reads only plain Go field access on the embedded +// *schedulerpb.SchedulerState proto and the serialized snapshot returned by +// Node.Snapshot. It never constructs a chasm.Context, because materializing a +// chasm Field (e.g. sched.Invoker.Get(ctx)) escalates the root node's +// valueState and marks the tree dirty even for a read. Only the idle-task +// validity check does, see idleTaskValid. + +// schedState is a test-only vocabulary for the scheduler's lifecycle. It is +// derived, never persisted: there is deliberately no state enum on the +// component, and adding one is a proto change with V1->V2 migration and +// rollback implications that this check does not require. +type schedState string + +const ( + // stateSentinel is a schedule-ID reservation. Inert by design. + stateSentinel schedState = "sentinel" + // stateMigrating is mid-migration to a workflow-backed (V1) scheduler, + // driven by SchedulerMigrateToWorkflowTask. + stateMigrating schedState = "migrating" + // stateCompleted is terminal: Closed is set. + stateCompleted schedState = "completed" + // stateIdle is non-terminal and quiescent, awaiting its idle timer. + stateIdle schedState = "idle" + // stateRunning is non-terminal with pending work armed. + stateRunning schedState = "running" + // stateStuck is the bug: non-terminal, but nothing will ever wake it. + stateStuck schedState = "stuck" +) + +// StateInvariant asserts the stuck and idle-deadline invariants against a clean +// tree. Build one with [NewStateInvariant]; [NewEngine] installs one on the +// engine it builds and exposes it as [SchedulerEngine.State]. +type StateInvariant struct { + registry *chasm.Registry + idleHandler *scheduler.SchedulerIdleTaskHandler + + mu sync.Mutex + allowStuckReason string +} + +// NewStateInvariant builds the check for trees registered in registry. +// config and logger back the idle task handler used to validate armed idle +// tasks. +func NewStateInvariant(registry *chasm.Registry, config *scheduler.Config, logger log.Logger) *StateInvariant { + return &StateInvariant{ + registry: registry, + idleHandler: scheduler.NewSchedulerIdleTaskHandler(scheduler.SchedulerIdleTaskHandlerOptions{ + Config: config, MetricsHandler: metrics.NoopMetricsHandler, BaseLogger: logger, + }), + } +} + +// AllowStuck opts out of both checks, for tests that deliberately drive the +// scheduler into a state with no pending work. +// +// reason is mandatory: an unexplained opt-out is indistinguishable from a +// silently tolerated bug, and the set of tests that need one is itself a +// finding worth reviewing. +func (c *StateInvariant) AllowStuck(t chasmtest.TB, reason string) { + t.Helper() + if reason == "" { + t.Fatal("AllowStuck requires a reason explaining why this test tolerates a stuck scheduler") + } + c.mu.Lock() + defer c.mu.Unlock() + c.allowStuckReason = reason +} + +// Check fails t if the scheduler in a clean tree is stuck, or advertises an idle +// deadline that no valid idle task backs. node must be the root and the tree +// must be clean; both hold immediately after a successful CloseTransaction. +func (c *StateInvariant) Check(t chasmtest.TB, node *chasm.Node, sched *scheduler.Scheduler) { + t.Helper() + if vs := c.Violations(node, sched); len(vs) > 0 { + t.Fatalf("%s", strings.Join(vs, "\n")) + } +} + +// Violations reports every broken predicate without failing a test, so a test +// that corrupts state on purpose can assert the check catches it. It reports +// nothing once [StateInvariant.AllowStuck] was called. +func (c *StateInvariant) Violations(node *chasm.Node, sched *scheduler.Scheduler) []string { + c.mu.Lock() + allowed := c.allowStuckReason != "" + c.mu.Unlock() + if allowed { + return nil + } + + var out []string + total, byPath := liveTaskCount(node) + if derivedState(sched, total) == stateStuck { + out = append(out, fmt.Sprintf("scheduler is stuck: %s", describeSched(sched, total, byPath))) + } + if v := c.idleViolation(node, sched, total, byPath); v != "" { + out = append(out, v) + } + return out +} + +// idleViolation asserts that a declared idle deadline is backed by an armed +// idle task that its handler still considers valid. +// +// IdleCloseTime is surfaced as the ScheduleIdleCloseTime search attribute and +// is what the production stuck-schedule scanner reads. When +// SchedulerIdleTaskHandler.Validate drops a task as expiration_shift and +// nothing arms a replacement, IdleCloseTime is left behind: the schedule never +// closes, and the very signal built to detect that reports a close time that +// will never arrive. Counting armed tasks is not enough, since an idle task the +// handler drops (deadline moved, schedule held open) leaves the same hole, so +// the armed tasks are run through Validate. +// +// Only asserted in this direction. Sentinels legitimately arm an idle task +// without setting IdleCloseTime (see NewSentinel), so an armed task with no +// declared deadline is not a defect. +func (c *StateInvariant) idleViolation(node *chasm.Node, sched *scheduler.Scheduler, total int, byPath map[string]int) string { + if sched.GetSentinel() || sched.GetClosed() || sched.GetIdleCloseTime() == nil { + return "" + } + if c.idleTaskValid(node, sched) { + return "" + } + return fmt.Sprintf("IdleCloseTime is set to %v but no valid SchedulerIdleTask is armed (%d armed), so the "+ + "schedule will never close and ScheduleIdleCloseTime reports a deadline that will never arrive: %s", + sched.GetIdleCloseTime().AsTime(), IdleTaskCount(c.registry, node), describeSched(sched, total, byPath)) +} + +// idleTaskValid reports whether any armed idle task passes the handler's +// Validate. It validates through a read-only context over the clean tree. +func (c *StateInvariant) idleTaskValid(node *chasm.Node, sched *scheduler.Scheduler) bool { + wantTypeID, ok := c.registry.TaskIDFor(&schedulerpb.SchedulerIdleTask{}) + if !ok { + return false + } + ctx := chasm.NewContext(context.Background(), node) + for _, n := range node.Snapshot(nil).Nodes { + for _, task := range n.GetMetadata().GetComponentAttributes().GetPureTasks() { + if task.GetTypeId() != wantTypeID { + continue + } + value := &schedulerpb.SchedulerIdleTask{} + if err := proto.Unmarshal(task.GetData().GetData(), value); err != nil { + return false + } + valid, err := c.idleHandler.Validate(ctx, sched, chasm.TaskInvocation{ + TaskAttributes: chasm.TaskAttributes{ScheduledTime: task.GetScheduledTime().AsTime()}, + }, value) + if err == nil && valid { + return true + } + } + } + return false +} + +// liveTaskCount counts logical tasks across every node in the tree. +// +// This deliberately counts *logical* tasks, from ChasmComponentAttributes. +// PureTasks and SideEffectTasks, rather than physical tasks. A physical task +// can exist on the backend while the logical task it points at has been reaped +// or invalidated, so counting physical tasks reports a schedule as armed when +// it is in fact stuck. Side-effect tasks count because some valid states, such +// as migration callback attachment, rely on one before arming pure tasks. +// (chasmtest.Engine.Tasks and testEnv.NodeBackend.TasksByCategory both expose +// physical tasks, and are the wrong source for this check.) +// +// Note that the framework-managed visibility task counts here, as it always +// has. A test engine never dispatches it, so it stays in the tree after any +// transition that touches a search attribute and masks a scheduler with nothing +// else left to run. Excluding it exposes a hand-built taskless state in +// TestGeneratorTask_EngineHighWaterMarkInvalidation, so the lifecycle driver +// keeps its own no-stuck rule, which ignores it (see CheckInvariants). +// +// node must be the root and the tree must be clean; Node.Snapshot panics +// otherwise. Both hold immediately after a successful CloseTransaction. +func liveTaskCount(node *chasm.Node) (int, map[string]int) { + byPath := make(map[string]int) + total := 0 + for path, n := range node.Snapshot(nil).Nodes { + attrs := n.GetMetadata().GetComponentAttributes() + taskCount := len(attrs.GetPureTasks()) + len(attrs.GetSideEffectTasks()) + if taskCount == 0 { + continue + } + byPath[path] = taskCount + total += taskCount + } + return total, byPath +} + +// derivedState computes the scheduler's lifecycle state from the tuple that +// actually represents it: (Sentinel, WorkflowMigration, Closed, IdleCloseTime, +// live logical task count). +func derivedState(sched *scheduler.Scheduler, liveTasks int) schedState { + switch { + case sched.GetSentinel(): + return stateSentinel + case sched.GetWorkflowMigration() != nil: + return stateMigrating + case sched.GetClosed(): + return stateCompleted + case liveTasks == 0: + return stateStuck + case sched.GetIdleCloseTime() != nil: + return stateIdle + default: + return stateRunning + } +} + +// IdleTaskCount counts armed SchedulerIdleTasks across the tree, resolving the +// task's registered type ID rather than matching on node path (the root +// Scheduler component also carries callback and migration tasks). It returns -1 +// if the idle task is not registered. +func IdleTaskCount(registry *chasm.Registry, node *chasm.Node) int { + wantTypeID, ok := registry.TaskIDFor(&schedulerpb.SchedulerIdleTask{}) + if !ok { + return -1 + } + count := 0 + for _, n := range node.Snapshot(nil).Nodes { + for _, task := range n.GetMetadata().GetComponentAttributes().GetPureTasks() { + if task.GetTypeId() == wantTypeID { + count++ + } + } + } + return count +} + +// describeSched renders the full state tuple for a failure message, so a +// triage does not require re-running under a debugger. +func describeSched(sched *scheduler.Scheduler, total int, byPath map[string]int) string { + paths := make([]string, 0, len(byPath)) + for p := range byPath { + paths = append(paths, fmt.Sprintf("%s=%d", p, byPath[p])) + } + slices.Sort(paths) + + var b strings.Builder + fmt.Fprintf(&b, "\n scheduleID: %s", sched.GetScheduleId()) + fmt.Fprintf(&b, "\n closed: %v", sched.GetClosed()) + fmt.Fprintf(&b, "\n sentinel: %v", sched.GetSentinel()) + fmt.Fprintf(&b, "\n paused: %v", sched.GetSchedule().GetState().GetPaused()) + fmt.Fprintf(&b, "\n limitedActions: %v", sched.GetSchedule().GetState().GetLimitedActions()) + fmt.Fprintf(&b, "\n remaining: %d", sched.GetSchedule().GetState().GetRemainingActions()) + fmt.Fprintf(&b, "\n conflictToken: %d", sched.GetConflictToken()) + fmt.Fprintf(&b, "\n idleCloseTime: %v", sched.GetIdleCloseTime().AsTime()) + fmt.Fprintf(&b, "\n migration: %v", sched.GetWorkflowMigration() != nil) + fmt.Fprintf(&b, "\n liveTasks: %d [%s]", total, strings.Join(paths, " ")) + return b.String() +} diff --git a/chasm/lib/scheduler/schedulertest/state_invariant_test.go b/chasm/lib/scheduler/schedulertest/state_invariant_test.go new file mode 100644 index 00000000000..71351b521b7 --- /dev/null +++ b/chasm/lib/scheduler/schedulertest/state_invariant_test.go @@ -0,0 +1,82 @@ +package schedulertest + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.temporal.io/server/chasm/lib/scheduler" + schedulerpb "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// TestDerivedState pins the classification tuple. Without this, a refactor +// that made derivedState never return stateStuck would silently disarm every +// stuckness assertion in the package while leaving the suite green. +func TestDerivedState(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + state *schedulerpb.SchedulerState + liveTasks int + want schedState + }{ + { + name: "open with an armed task is running", + state: &schedulerpb.SchedulerState{}, + liveTasks: 1, + want: stateRunning, + }, + { + name: "open with an armed task and an idle deadline is idle", + state: &schedulerpb.SchedulerState{IdleCloseTime: timestamppb.New(time.Unix(100, 0))}, + liveTasks: 1, + want: stateIdle, + }, + { + name: "open with no armed task is stuck", + state: &schedulerpb.SchedulerState{}, + liveTasks: 0, + want: stateStuck, + }, + { + name: "an idle deadline does not excuse a missing task", + state: &schedulerpb.SchedulerState{IdleCloseTime: timestamppb.New(time.Unix(100, 0))}, + liveTasks: 0, + want: stateStuck, + }, + { + name: "closed with no armed task is terminal, not stuck", + state: &schedulerpb.SchedulerState{Closed: true}, + liveTasks: 0, + want: stateCompleted, + }, + { + name: "sentinel is inert by design", + state: &schedulerpb.SchedulerState{Sentinel: true}, + liveTasks: 0, + want: stateSentinel, + }, + { + name: "mid-migration is excused", + state: &schedulerpb.SchedulerState{WorkflowMigration: &schedulerpb.WorkflowMigrationState{}}, + liveTasks: 0, + want: stateMigrating, + }, + { + name: "closed takes precedence over a stale idle deadline", + state: &schedulerpb.SchedulerState{Closed: true, IdleCloseTime: timestamppb.New(time.Unix(100, 0))}, + liveTasks: 0, + want: stateCompleted, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + sched := &scheduler.Scheduler{SchedulerState: tc.state} + require.Equal(t, tc.want, derivedState(sched, tc.liveTasks)) + }) + } +} diff --git a/chasm/lib/scheduler/schedulertest/task_validity_test.go b/chasm/lib/scheduler/schedulertest/task_validity_test.go new file mode 100644 index 00000000000..bea44c7e8cb --- /dev/null +++ b/chasm/lib/scheduler/schedulertest/task_validity_test.go @@ -0,0 +1,111 @@ +package schedulertest + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + enumspb "go.temporal.io/api/enums/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/scheduler" + schedulerpb "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" + "go.temporal.io/server/common" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestStalePhysicalTaskIsNotProgress(t *testing.T) { + t.Parallel() + schedule := DefaultSchedule() + schedule.State.Paused = true + schedule.Spec.Interval = nil + d := NewDriver(t, WithSchedule(schedule)) + require.NoError(t, d.TriggerImmediately()) + require.NotEmpty(t, d.runner.PendingSideEffectTasks()) + _, err := d.Engine.UpdateComponent(chasm.NewEngineContext(context.Background(), d.Engine), d.Ref, + func(ctx chasm.MutableContext, c chasm.Component) error { + c.(*scheduler.Scheduler).Invoker.Get(ctx).BufferedStarts = nil + return nil + }) + require.NoError(t, err) + require.NotEmpty(t, d.runner.PendingSideEffectTasks(), "physical deliveries survive logical invalidation") + require.False(t, d.HasPendingTask()) + require.False(t, d.StepSideEffect()) + require.Empty(t, d.Snapshot().AcceptedStarts) +} + +func TestOldExecuteTaskCanProcessCurrentWork(t *testing.T) { + t.Parallel() + schedule := DefaultSchedule() + schedule.State.Paused = true + schedule.Policies.OverlapPolicy = enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL + d := NewDriver(t, WithSchedule(schedule)) + require.NoError(t, d.TriggerImmediately()) + tasks := d.runner.PendingSideEffectTasks() + require.NotEmpty(t, tasks) + old := tasks[0] + _, err := d.Engine.UpdateComponent(chasm.NewEngineContext(context.Background(), d.Engine), d.Ref, + func(ctx chasm.MutableContext, c chasm.Component) error { + inv := c.(*scheduler.Scheduler).Invoker.Get(ctx) + current := common.CloneProto(inv.BufferedStarts[0]) + current.RequestId = "current-request" + current.WorkflowId = "current-workflow" + inv.BufferedStarts = append(inv.BufferedStarts, current) + inv.BufferedStarts[0].RunId = "original-run" + return nil + }) + require.NoError(t, err) + inTree, valid := d.runner.ValidateSideEffectTask(old) + require.True(t, inTree) + require.True(t, valid) + require.True(t, d.StepSideEffect()) + require.Contains(t, d.Snapshot().AcceptedStarts, "current-request") +} + +// TestIdleDeadlineRequiresValidIdleTask corrupts the advertised idle deadline so +// the armed idle task no longer matches it, and asserts the shared state +// invariant (which validates armed idle tasks through the handler) reports it. +// The driver is built WithoutStateInvariant, since the engine would otherwise +// fail the test the moment the corrupting transition commits. +func TestIdleDeadlineRequiresValidIdleTask(t *testing.T) { + t.Parallel() + schedule := DefaultSchedule() + schedule.State.LimitedActions = true + d := NewDriver(t, WithSchedule(schedule), WithoutStateInvariant()) + require.Empty(t, d.StateViolations()) + _, err := d.Engine.UpdateComponent(chasm.NewEngineContext(context.Background(), d.Engine), d.Ref, + func(ctx chasm.MutableContext, c chasm.Component) error { + s := c.(*scheduler.Scheduler) + s.LastEventTime = timestamppb.New(d.Now().Add(time.Hour)) + s.IdleCloseTime = timestamppb.New(s.GetIdleCloseTime().AsTime().Add(time.Hour)) + // Deliberately omit Generator.Generate, which publishes the replacement. + return nil + }) + require.NoError(t, err) + violations := d.StateViolations() + require.Len(t, violations, 1) + require.Contains(t, violations[0], "no valid SchedulerIdleTask is armed") +} + +func TestRetryTaskEqualityAndRearming(t *testing.T) { + t.Parallel() + schedule := DefaultSchedule() + schedule.State.Paused = true + d := NewDriver(t, WithSchedule(schedule)) + require.NoError(t, d.TriggerImmediately()) + deadline := d.Now().Add(time.Minute) + _, err := d.Engine.UpdateComponent(chasm.NewEngineContext(context.Background(), d.Engine), d.Ref, + func(ctx chasm.MutableContext, c chasm.Component) error { + inv := c.(*scheduler.Scheduler).Invoker.Get(ctx) + inv.BufferedStarts[0].Attempt = 2 + inv.BufferedStarts[0].BackoffTime = timestamppb.New(deadline) + ctx.AddTask(inv, chasm.TaskAttributes{ScheduledTime: deadline}, &schedulerpb.InvokerProcessBufferTask{}) + return nil + }) + require.NoError(t, err) + require.False(t, d.StepSideEffect()) + require.True(t, d.StepPure()) + require.Equal(t, deadline, d.Now()) + require.True(t, d.StepSideEffect()) + require.Len(t, d.Snapshot().AcceptedStarts, 1) +} diff --git a/chasm/lib/scheduler/spec_processor_properties_test.go b/chasm/lib/scheduler/spec_processor_properties_test.go new file mode 100644 index 00000000000..f93d18b428d --- /dev/null +++ b/chasm/lib/scheduler/spec_processor_properties_test.go @@ -0,0 +1,199 @@ +package scheduler_test + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + enumspb "go.temporal.io/api/enums/v1" + schedulespb "go.temporal.io/server/api/schedule/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/scheduler" + "go.temporal.io/server/common/clock" + "go.uber.org/mock/gomock" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/durationpb" +) + +func TestProcessTimeRange_ManualPaginationPreservesOccurrences(t *testing.T) { + for _, jitter := range []time.Duration{0, 15 * time.Second, 59 * time.Second, 2 * time.Minute} { + for _, pageSize := range []int{1, 2, 5, 20} { + t.Run(fmt.Sprintf("jitter=%s/page=%d", jitter, pageSize), func(t *testing.T) { + env := newTestEnv(t) + ctx := chasm.NewMutableContext(context.Background(), env.Node) + schedule := defaultSchedule() + schedule.Spec.Jitter = durationpb.New(jitter) + sched, err := scheduler.NewScheduler(ctx, namespace, namespaceID, scheduleID, schedule, nil) + require.NoError(t, err) + processor := newTestSpecProcessor(env.Ctrl) + start := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC) + end := start.Add(17*time.Minute + 59*time.Second) + whole, err := processor.ProcessTimeRange(sched, start, end, + enumspb.SCHEDULE_OVERLAP_POLICY_UNSPECIFIED, sched.WorkflowID(), "backfill", true, nil) + require.NoError(t, err) + require.Len(t, whole.BufferedStarts, 17) + for i, action := range whole.BufferedStarts { + nominal := start.Add(time.Duration(i+1) * time.Minute) + require.Equal(t, nominal, action.NominalTime.AsTime()) + require.False(t, action.ActualTime.AsTime().Before(nominal)) + require.False(t, action.ActualTime.AsTime().After(nominal.Add(jitter))) + require.True(t, action.ActualTime.AsTime().Before(nominal.Add(time.Minute)), "jitter must leave the successor enumerable") + } + + var paged []*schedulespb.BufferedStart + cursor := start + for page := 0; ; page++ { + require.LessOrEqual(t, page, len(whole.BufferedStarts), "pagination failed to terminate") + limit := pageSize + part, err := processor.ProcessTimeRange(sched, cursor, end, + enumspb.SCHEDULE_OVERLAP_POLICY_UNSPECIFIED, sched.WorkflowID(), "backfill", true, &limit) + require.NoError(t, err) + require.Zero(t, part.DroppedCount) + require.Equal(t, pageSize-len(part.BufferedStarts), limit) + paged = append(paged, part.BufferedStarts...) + if len(part.BufferedStarts) == 0 { + break + } + require.True(t, part.LastActionTime.After(cursor), "cursor must make progress") + require.Equal(t, part.BufferedStarts[len(part.BufferedStarts)-1].ActualTime.AsTime(), part.LastActionTime) + cursor = part.LastActionTime + } + + require.Len(t, paged, len(whole.BufferedStarts)) + for i, action := range whole.BufferedStarts { + require.True(t, proto.Equal(action, paged[i]), "occurrence %d changed during pagination", i) + } + }) + } + } +} + +func TestProcessTimeRange_AutomatedCapacityAccounting(t *testing.T) { + for _, occurrences := range []int{1, 2, 7, 17} { + for _, capacity := range []int{-1, 0, 1, 3, 20} { + t.Run(fmt.Sprintf("occurrences=%d/capacity=%d", occurrences, capacity), func(t *testing.T) { + env := newTestEnv(t) + ctx := chasm.NewMutableContext(context.Background(), env.Node) + schedule := defaultSchedule() + schedule.Policies.CatchupWindow = durationpb.New(time.Hour) + sched, err := scheduler.NewScheduler(ctx, namespace, namespaceID, scheduleID, schedule, nil) + require.NoError(t, err) + processor := newTestSpecProcessor(env.Ctrl) + start := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC) + end := start.Add(time.Duration(occurrences) * time.Minute) + limit := capacity + result, err := processor.ProcessTimeRange(sched, start, end, + enumspb.SCHEDULE_OVERLAP_POLICY_UNSPECIFIED, sched.WorkflowID(), "", false, &limit) + require.NoError(t, err) + buffered := min(occurrences, max(capacity, 0)) + require.Len(t, result.BufferedStarts, buffered) + require.Equal(t, int64(occurrences-buffered), result.DroppedCount) + require.Equal(t, capacity-buffered, limit) + require.Equal(t, end, result.LastActionTime) + require.Equal(t, end.Add(time.Minute), result.NextWakeupTime) + for i, action := range result.BufferedStarts { + want := start.Add(time.Duration(i+1) * time.Minute) + require.Equal(t, want, action.NominalTime.AsTime()) + require.Equal(t, want, action.ActualTime.AsTime()) + } + + resumed, err := processor.ProcessTimeRange(sched, result.LastActionTime, end.Add(time.Minute), + enumspb.SCHEDULE_OVERLAP_POLICY_UNSPECIFIED, sched.WorkflowID(), "", false, nil) + require.NoError(t, err) + require.Len(t, resumed.BufferedStarts, 1) + require.Equal(t, end.Add(time.Minute), resumed.BufferedStarts[0].ActualTime.AsTime()) + }) + } + } +} + +func TestProcessTimeRange_JitterCursorBoundaries(t *testing.T) { + env := newTestEnv(t) + ctx := chasm.NewMutableContext(context.Background(), env.Node) + schedule := defaultSchedule() + schedule.Spec.Jitter = durationpb.New(59 * time.Second) + sched, err := scheduler.NewScheduler(ctx, namespace, namespaceID, scheduleID, schedule, nil) + require.NoError(t, err) + processor := newTestSpecProcessor(env.Ctrl) + start := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC) + first, err := processor.NextTime(sched, start) + require.NoError(t, err) + require.Equal(t, start.Add(time.Minute), first.Nominal) + require.True(t, first.Next.After(first.Nominal), "fixture must exercise positive jitter") + require.True(t, first.Next.Before(start.Add(2*time.Minute))) + + t.Run("actual end is inclusive", func(t *testing.T) { + result, err := processor.ProcessTimeRange(sched, start, first.Next, + enumspb.SCHEDULE_OVERLAP_POLICY_UNSPECIFIED, sched.WorkflowID(), "backfill", true, nil) + require.NoError(t, err) + require.Len(t, result.BufferedStarts, 1) + require.Equal(t, first.Nominal, result.BufferedStarts[0].NominalTime.AsTime()) + require.Equal(t, first.Next, result.BufferedStarts[0].ActualTime.AsTime()) + require.Equal(t, first.Next, result.LastActionTime) + + resumed, err := processor.ProcessTimeRange(sched, result.LastActionTime, start.Add(3*time.Minute), + enumspb.SCHEDULE_OVERLAP_POLICY_UNSPECIFIED, sched.WorkflowID(), "backfill", true, nil) + require.NoError(t, err) + require.Len(t, resumed.BufferedStarts, 1) + require.Equal(t, start.Add(2*time.Minute), resumed.BufferedStarts[0].NominalTime.AsTime()) + }) + + t.Run("unvisited jittered occurrence remains enumerable", func(t *testing.T) { + boundary := first.Next.Add(-time.Nanosecond) + before, err := processor.ProcessTimeRange(sched, start, boundary, + enumspb.SCHEDULE_OVERLAP_POLICY_UNSPECIFIED, sched.WorkflowID(), "backfill", true, nil) + require.NoError(t, err) + require.Empty(t, before.BufferedStarts) + require.Equal(t, start, before.LastActionTime) + require.Equal(t, first.Next, before.NextWakeupTime) + + resumed, err := processor.ProcessTimeRange(sched, before.LastActionTime, first.Next, + enumspb.SCHEDULE_OVERLAP_POLICY_UNSPECIFIED, sched.WorkflowID(), "backfill", true, nil) + require.NoError(t, err) + require.Len(t, resumed.BufferedStarts, 1) + require.Equal(t, first.Nominal, resumed.BufferedStarts[0].NominalTime.AsTime()) + require.Equal(t, first.Next, resumed.BufferedStarts[0].ActualTime.AsTime()) + }) +} + +func TestGeneratorTask_JitterPendingOccurrenceSurvivesImmediateGeneration(t *testing.T) { + base := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC) + ts := clock.NewEventTimeSource() + ts.Update(base) + schedule := defaultSchedule() + schedule.Spec.Jitter = durationpb.New(59 * time.Second) + e := newSchedulerTestEngine(t, schedule, withEngineTimeSource(ts)) + _, err := e.engine.FirePureTasks(e.rootRef, base) + require.NoError(t, err) + processor := newRealSpecProcessor(gomock.NewController(t), e.logger) + var actual time.Time + require.NoError(t, e.readScheduler(func(s *scheduler.Scheduler, _ chasm.Context) error { + next, err := processor.NextTime(s, base) + if err != nil { + return err + } + require.Equal(t, base.Add(time.Minute), next.Nominal) + require.True(t, next.Next.After(next.Nominal)) + actual = next.Next + return nil + })) + ts.Update(actual.Add(-time.Nanosecond)) + require.NoError(t, e.updateScheduler(func(s *scheduler.Scheduler, ctx chasm.MutableContext) error { + s.Generator.Get(ctx).Generate(ctx) + return nil + })) + _, err = e.engine.FirePureTasks(e.rootRef, ts.Now()) + require.NoError(t, err) + ts.Update(actual) + _, err = e.engine.FirePureTasks(e.rootRef, actual) + require.NoError(t, err) + require.NoError(t, e.readScheduler(func(s *scheduler.Scheduler, ctx chasm.Context) error { + starts := s.Invoker.Get(ctx).BufferedStarts + require.Len(t, starts, 1, "immediate generation must retain the pending jittered occurrence") + require.Equal(t, base.Add(time.Minute), starts[0].NominalTime.AsTime()) + require.Equal(t, actual, starts[0].ActualTime.AsTime()) + return nil + })) +} diff --git a/chasm/lib/scheduler/task_validity_engine_test.go b/chasm/lib/scheduler/task_validity_engine_test.go new file mode 100644 index 00000000000..e5d9ec5ec5f --- /dev/null +++ b/chasm/lib/scheduler/task_validity_engine_test.go @@ -0,0 +1,276 @@ +package scheduler_test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.temporal.io/api/workflowservice/v1" + schedulespb "go.temporal.io/server/api/schedule/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/chasmtest" + "go.temporal.io/server/chasm/lib/scheduler" + "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" + "go.temporal.io/server/common/clock" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/metrics" + "go.uber.org/mock/gomock" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestGeneratorTask_EngineHighWaterMarkInvalidation(t *testing.T) { + env := newSchedulerTestEngine(t, defaultSchedule()) + handler := newGeneratorTaskHandlerForValidity(t, env.logger) + now := env.timeSource.Now() + taskTime := now.Add(time.Minute) + task := &schedulerpb.GeneratorTask{} + + var generator *scheduler.Generator + err := env.updateScheduler( + func(s *scheduler.Scheduler, ctx chasm.MutableContext) error { + generator = s.Generator.Get(ctx) + generator.LastProcessedTime = timestamppb.New(now) + ctx.AddTask(generator, chasm.TaskAttributes{ScheduledTime: taskTime}, task) + return nil + }) + require.NoError(t, err) + + err = env.updateScheduler( + func(s *scheduler.Scheduler, ctx chasm.MutableContext) error { + generator = s.Generator.Get(ctx) + s.Info.OverlapSkipped++ // unrelated scheduler state + valid, err := handler.Validate(ctx, generator, + chasm.TaskInvocation{TaskAttributes: chasm.TaskAttributes{ScheduledTime: taskTime}}, task) + if err != nil { + return err + } + require.True(t, valid) + generator.LastProcessedTime = timestamppb.New(taskTime) + return nil + }) + require.NoError(t, err) + + dropped, err := chasmtest.ExecutePureTask( + context.Background(), env.engine, generator, handler, chasm.TaskAttributes{ScheduledTime: taskTime}, task) + require.NoError(t, err) + require.True(t, dropped) + + dropped, err = chasmtest.ExecutePureTask( + context.Background(), env.engine, generator, handler, chasm.TaskAttributes{}, task) + require.NoError(t, err) + require.False(t, dropped, "immediate Generator tasks ignore the high water mark") +} + +func newGeneratorTaskHandlerForValidity(t *testing.T, logger log.Logger) *scheduler.GeneratorTaskHandler { + t.Helper() + specProcessor := newRealSpecProcessor(gomock.NewController(t), logger) + return scheduler.NewGeneratorTaskHandler(scheduler.GeneratorTaskHandlerOptions{ + Config: defaultConfig(), + MetricsHandler: metrics.NoopMetricsHandler, + BaseLogger: logger, + SpecProcessor: specProcessor, + SpecBuilder: newLegacySpecBuilder(0, 0), + }) +} + +func TestIdleTask_EngineActivityInvalidatesAndRearms(t *testing.T) { + timeSource := clock.NewEventTimeSource() + now := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) + timeSource.Update(now) + schedule := defaultSchedule() + schedule.State.LimitedActions = true + schedule.State.RemainingActions = 0 + env := newSchedulerTestEngine( + t, schedule, withEngineTimeSource(timeSource)) + handler := scheduler.NewSchedulerIdleTaskHandler(scheduler.SchedulerIdleTaskHandlerOptions{ + Config: defaultConfig(), + MetricsHandler: metrics.NoopMetricsHandler, + BaseLogger: env.logger, + }) + + var oldDeadline time.Time + err := env.readScheduler( + func(s *scheduler.Scheduler, _ chasm.Context) error { + oldDeadline = s.GetIdleCloseTime().AsTime() + return nil + }) + require.NoError(t, err) + require.False(t, oldDeadline.IsZero()) + idleTime := oldDeadline.Sub(now) + err = env.updateScheduler( + func(s *scheduler.Scheduler, _ chasm.MutableContext) error { + s.Info.OverlapSkipped++ // unrelated scheduler state + return nil + }) + require.NoError(t, err) + + err = env.readScheduler( + func(s *scheduler.Scheduler, ctx chasm.Context) error { + valid, err := handler.Validate(ctx, s, + chasm.TaskInvocation{TaskAttributes: chasm.TaskAttributes{ScheduledTime: oldDeadline}}, + &schedulerpb.SchedulerIdleTask{IdleTimeTotal: durationpb.New(idleTime)}) + if err != nil { + return err + } + require.True(t, valid) + return nil + }) + require.NoError(t, err) + + timeSource.Update(now.Add(time.Minute)) + _, err = scheduler.NewTestHandler(env.logger).UpdateSchedule(env.engineCtx, &schedulerpb.UpdateScheduleRequest{ + NamespaceId: namespaceID, + FrontendRequest: &workflowservice.UpdateScheduleRequest{ + Namespace: namespace, + ScheduleId: scheduleID, + Schedule: schedule, + }, + }) + require.NoError(t, err) + + var sched *scheduler.Scheduler + var newDeadline time.Time + err = env.readScheduler( + func(s *scheduler.Scheduler, ctx chasm.Context) error { + sched = s + newDeadline = s.GetIdleCloseTime().AsTime() + valid, err := handler.Validate(ctx, s, + chasm.TaskInvocation{TaskAttributes: chasm.TaskAttributes{ScheduledTime: newDeadline}}, + &schedulerpb.SchedulerIdleTask{IdleTimeTotal: durationpb.New(idleTime)}) + if err != nil { + return err + } + require.True(t, valid) + return nil + }) + require.NoError(t, err) + require.True(t, newDeadline.After(oldDeadline)) + + dropped, err := chasmtest.ExecutePureTask( + context.Background(), env.engine, sched, handler, + chasm.TaskAttributes{ScheduledTime: oldDeadline}, + &schedulerpb.SchedulerIdleTask{IdleTimeTotal: durationpb.New(idleTime)}) + require.NoError(t, err) + require.True(t, dropped) +} + +func TestCallbacksTask_EngineCurrentTupleInvalidation(t *testing.T) { + tests := []struct { + name string + mutate func(*schedulespb.BufferedStart) + }{ + {name: "callback recorded", mutate: func(start *schedulespb.BufferedStart) { start.HasCallback = true }}, + {name: "run ID cleared", mutate: func(start *schedulespb.BufferedStart) { start.RunId = "" }}, + {name: "completion recorded", mutate: func(start *schedulespb.BufferedStart) { + start.Completed = &schedulespb.CompletedResult{} + }}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + env := newSchedulerTestEngine(t, defaultSchedule()) + handler := scheduler.NewSchedulerCallbacksTaskHandler(scheduler.SchedulerCallbacksTaskHandlerOptions{ + Config: defaultConfig(), + }) + task := &schedulerpb.SchedulerCallbacksTask{} + + var sched *scheduler.Scheduler + err := env.updateScheduler( + func(s *scheduler.Scheduler, ctx chasm.MutableContext) error { + sched = s + s.Invoker.Get(ctx).BufferedStarts = []*schedulespb.BufferedStart{{ + RequestId: "running-work", + WorkflowId: "running-workflow", + RunId: "running-run", + Attempt: 1, + }} + ctx.AddTask(s, chasm.TaskAttributes{}, task) + return nil + }) + require.NoError(t, err) + + err = env.updateScheduler( + func(s *scheduler.Scheduler, ctx chasm.MutableContext) error { + sched = s + s.Info.OverlapSkipped++ // unrelated scheduler state + valid, err := handler.Validate(ctx, s, chasm.TaskInvocation{}, task) + if err != nil { + return err + } + require.True(t, valid) + test.mutate(s.Invoker.Get(ctx).BufferedStarts[0]) + return nil + }) + require.NoError(t, err) + + dropped, err := chasmtest.ExecuteSideEffectTask( + context.Background(), env.engine, sched, handler, chasm.TaskAttributes{}, task) + require.NoError(t, err) + require.True(t, dropped) + }) + } +} + +func TestMigrationTask_EngineStateInvalidation(t *testing.T) { + tests := []struct { + name string + mutate func(*scheduler.Scheduler) + }{ + { + name: "migration cleared", + mutate: func(s *scheduler.Scheduler) { + s.WorkflowMigration = nil + }, + }, + { + name: "scheduler closed", + mutate: func(s *scheduler.Scheduler) { + s.Closed = true + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + env := newSchedulerTestEngine(t, defaultSchedule()) + handler := scheduler.NewSchedulerMigrateToWorkflowTaskHandler( + scheduler.SchedulerMigrateToWorkflowTaskHandlerOptions{ + Config: defaultConfig(), + MetricsHandler: metrics.NoopMetricsHandler, + BaseLogger: env.logger, + }) + task := &schedulerpb.SchedulerMigrateToWorkflowTask{} + + var sched *scheduler.Scheduler + err := env.updateScheduler( + func(s *scheduler.Scheduler, ctx chasm.MutableContext) error { + sched = s + s.WorkflowMigration = &schedulerpb.WorkflowMigrationState{} + ctx.AddTask(s, chasm.TaskAttributes{}, task) + return nil + }) + require.NoError(t, err) + + err = env.updateScheduler( + func(s *scheduler.Scheduler, ctx chasm.MutableContext) error { + sched = s + s.Info.OverlapSkipped++ // unrelated scheduler state + valid, err := handler.Validate(ctx, s, chasm.TaskInvocation{}, task) + if err != nil { + return err + } + require.True(t, valid) + test.mutate(s) + return nil + }) + require.NoError(t, err) + + dropped, err := chasmtest.ExecuteSideEffectTask( + context.Background(), env.engine, sched, handler, chasm.TaskAttributes{}, task) + require.NoError(t, err) + require.True(t, dropped) + }) + } +} diff --git a/cmd/tools/mutationtest/main.go b/cmd/tools/mutationtest/main.go new file mode 100644 index 00000000000..366d51b581b --- /dev/null +++ b/cmd/tools/mutationtest/main.go @@ -0,0 +1,195 @@ +// mutationtest checks that targeted regression tests detect deliberate +// production-code mutations. For each mutation it builds a mutated copy of one +// production file, points `go test -overlay` at it, runs the named tests, and +// requires a genuine assertion failure (not a build failure, not a pass). +// +// Edits are described by anchors that must each match exactly once in the +// unmutated file, so a refactor that moves or renames the anchored code makes +// this tool fail loudly instead of silently testing nothing. +// +// Run from anywhere inside the repository: go run ./cmd/tools/mutationtest +package main + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" +) + +// edit describes a single textual change to a file. +// +// If End is empty, the text Begin is replaced by Replacement. +// Otherwise the region from the start of Begin up to (but not including) the +// start of End is replaced by Replacement; End must appear after Begin. +// Begin and End must each occur exactly once in the file. +type edit struct { + Begin string + End string + Replacement string +} + +type mutation struct { + Name string + File string // repo-relative path of the production file to mutate + Package string // package passed to go test + Test string // -run regex + Edit edit +} + +const ( + schedulerDir = "chasm/lib/scheduler/" + schedulerTest = "./chasm/lib/scheduler/schedulertest" +) + +var mutations = []mutation{ + { + // Replace the guard so the callback is attached before a RunId exists. + Name: "early-callback", + File: schedulerDir + "completion.go", + Package: schedulerTest, + Test: "TestStartAcceptanceCallbackAndRecording/AllowAll/before", + Edit: edit{ + Begin: "\tif !internal.TracksCompletionResult(start.GetOverlapPolicy()) && start.GetRunId() == \"\" {", + Replacement: "\tif start.GetCompleted() != nil {", + }, + }, + { + // Delete the code between the discarded-selection comment and the + // re-arm comment, so deferred starts are never re-armed. + Name: "deferred-rearm", + File: schedulerDir + "invoker.go", + Package: schedulerTest, + Test: "TestExpiredSelectionReleasesRemainingBuffer|TestLedger_DeferredStartsStrandedByPausedCompletion", + Edit: edit{ + Begin: "\t// A discarded selection cannot produce", + End: "\t// Re-arm tasks if this call changed state", + }, + }, + { + Name: "capacity-progress", + File: schedulerDir + "backfiller_tasks.go", + Package: "./chasm/lib/scheduler", + Test: "TestBackfillCapacityProgress", + Edit: edit{ + Begin: "return min(available, max(1, available/backfillerCount))", + Replacement: "return available / backfillerCount", + }, + }, + { + // Short-circuit the selected-start predicate so that selected starts + // awaiting a RunId no longer hold the overlap slot. + Name: "selected-start-reservation", + File: schedulerDir + "invoker_tasks.go", + Package: schedulerTest, + Test: "TestSelectedStartReservesOverlapSlot", + Edit: edit{ + Begin: "start.GetAttempt() > 0 && start.GetCompleted() == nil &&\n\t\t\t\t", + End: "internal.TracksCompletionResult(start.GetOverlapPolicy())\n\t\t})", + Replacement: "false && ", + }, + }, +} + +func main() { + if err := run(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run() error { + root, err := repoRoot() + if err != nil { + return err + } + for _, m := range mutations { + if err := runMutation(root, m); err != nil { + return fmt.Errorf("%s: %w", m.Name, err) + } + fmt.Printf("%s: detected\n", m.Name) + } + return nil +} + +func repoRoot() (string, error) { + dir, err := os.Getwd() + if err != nil { + return "", err + } + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return dir, nil + } + parent := filepath.Dir(dir) + if parent == dir { + return "", errors.New("go.mod not found above working directory") + } + dir = parent + } +} + +// apply returns text with e applied, or an error if any anchor does not match +// exactly once. +func apply(text string, e edit) (string, error) { + if n := strings.Count(text, e.Begin); n != 1 { + return "", fmt.Errorf("begin anchor matched %d times, want exactly 1: %q", n, e.Begin) + } + first := strings.Index(text, e.Begin) + if e.End == "" { + return text[:first] + e.Replacement + text[first+len(e.Begin):], nil + } + if n := strings.Count(text, e.End); n != 1 { + return "", fmt.Errorf("end anchor matched %d times, want exactly 1: %q", n, e.End) + } + last := strings.Index(text, e.End) + if last < first { + return "", fmt.Errorf("end anchor %q appears before begin anchor %q", e.End, e.Begin) + } + return text[:first] + e.Replacement + text[last:], nil +} + +func runMutation(root string, m mutation) error { + source := filepath.Join(root, m.File) + original, err := os.ReadFile(source) + if err != nil { + return err + } + mutated, err := apply(string(original), m.Edit) + if err != nil { + return fmt.Errorf("%s: %w", m.File, err) + } + + temp, err := os.MkdirTemp("", "scheduler-mutation-") + if err != nil { + return err + } + defer func() { _ = os.RemoveAll(temp) }() + + replacement := filepath.Join(temp, filepath.Base(m.File)) + if err := os.WriteFile(replacement, []byte(mutated), 0o644); err != nil { + return err + } + overlay, err := json.Marshal(map[string]map[string]string{"Replace": {source: replacement}}) + if err != nil { + return err + } + overlayPath := filepath.Join(temp, "overlay.json") + if err := os.WriteFile(overlayPath, overlay, 0o644); err != nil { + return err + } + + cmd := exec.Command("go", "test", "-tags", "test_dep", "-overlay", overlayPath, + m.Package, "-run", m.Test, "-count=1") + cmd.Dir = root + cmd.Env = append(os.Environ(), "CGO_ENABLED=0") + out, err := cmd.CombinedOutput() + output := string(out) + if err == nil || !strings.Contains(output, "--- FAIL: ") || strings.Contains(output, "[build failed]") { + return fmt.Errorf("mutation did not produce an assertion failure\n%s", output) + } + return nil +} diff --git a/cmd/tools/mutationtest/main_test.go b/cmd/tools/mutationtest/main_test.go new file mode 100644 index 00000000000..d918765e43b --- /dev/null +++ b/cmd/tools/mutationtest/main_test.go @@ -0,0 +1,32 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestApply(t *testing.T) { + const text = "a one b two c" + + got, err := apply(text, edit{Begin: "one", Replacement: "1"}) + require.NoError(t, err) + require.Equal(t, "a 1 b two c", got) + + got, err = apply(text, edit{Begin: "one", End: "two", Replacement: "X "}) + require.NoError(t, err) + require.Equal(t, "a X two c", got) + + got, err = apply(text, edit{Begin: "one", End: "two"}) + require.NoError(t, err) + require.Equal(t, "a two c", got) + + _, err = apply(text, edit{Begin: "missing"}) + require.Error(t, err) + _, err = apply("x x", edit{Begin: "x"}) + require.Error(t, err) + _, err = apply(text, edit{Begin: "one", End: "absent"}) + require.Error(t, err) + _, err = apply(text, edit{Begin: "two", End: "one"}) + require.Error(t, err) +} diff --git a/docs/architecture/chasm-schedulertest-coverage-plan.md b/docs/architecture/chasm-schedulertest-coverage-plan.md new file mode 100644 index 00000000000..885cf43677c --- /dev/null +++ b/docs/architecture/chasm-schedulertest-coverage-plan.md @@ -0,0 +1,195 @@ +# CHASM scheduler lifecycle driver coverage + +## Commit provenance + +Branch: `test/chasm-scheduler-lifecycle`. +Base: `03ecd1d228d2b8d9d433da55785afc9aeeead918`. +David Porter's [temporalio/temporal#13042](https://github.com/temporalio/temporal/pull/13042) +head `33ffb9e001299e3221e107ce781f832b4a616a93` is imported as one signed commit, +`aa47350a376ce3d993bebc069e862d2362bab8c5`. Its tree exactly matches that source +head. Its message records all twelve original commit IDs and preserves David's +authorship. The following signed commit contains the additions described here. + +## Coverage and reference PRs + +| Reference | Coverage on this branch | +| --- | --- | +| [#35](https://github.com/chaptersix/temporal/pull/35) | `task_validity_engine_test.go` ports generator, idle, callback-repair and migration validation cases. `schedulertest/task_validity_test.go` distinguishes stale physical deliveries from valid current work, checks retry equality/rearming, and detects an advertised idle deadline with no valid logical task. | +| [#41](https://github.com/chaptersix/temporal/pull/41) | `TestDeferredBufferOneProperty` adapts the occupancy/accounting property to actual handlers. Existing `TestCompletedHistoryDoesNotConsumeBackfillCapacity`, capacity arithmetic tests and duplicate-result tests remain part of the required package run; no experimental planner refactor is imported. | +| [#51](https://github.com/chaptersix/temporal/pull/51), [#52](https://github.com/chaptersix/temporal/pull/52) | Ports 450/451/1000-range finite-drain tests and the positive-capacity admission fix. The test supplies capacity by consuming each batch; it checks unique identities, bounds and termination, not cross-schedule fairness. | +| [#42](https://github.com/chaptersix/temporal/pull/42), [#43](https://github.com/chaptersix/temporal/pull/43) | Native exclusive-cursor control plus migrated cursor/epoch and callback-reconciliation regressions in `migration_backfill_test.go`, with the cursor and reconciliation-ordering fixes. Watermark presence replaces the separate progress bit, so no schema change is imported. | +| [#53](https://github.com/chaptersix/temporal/pull/53), [#55](https://github.com/chaptersix/temporal/pull/55) | Native phase controls, conversion evidence, rollback phase regressions and the rollback production gate. The driver reports execution/retry/cancel/terminate obligations made unexecutable by migration; it no longer exempts every migrating state from its ledger. | +| [#59](https://github.com/chaptersix/temporal/pull/59), [#62](https://github.com/chaptersix/temporal/pull/62) | Controlled acceptance/response events, buffer reordering during an outstanding RPC, repeated result reconciliation, and directed selected-start reservation regressions with a minimal reservation fix. Workflow scenarios are adapted without importing experimental activity APIs or action schemas. | +| [#70](https://github.com/chaptersix/temporal/pull/70) | Duplicate callbacks before/after local start recording for ALLOW_ALL, SKIP and BUFFER_ALL, plus the early ALLOW_ALL callback guard. This branch retains callback attachment; the separate rollout-dependent callback-omission change is not imported. | +| [#44](https://github.com/chaptersix/temporal/pull/44), [#45](https://github.com/chaptersix/temporal/pull/45) | `migration_handoff_test.go` and the V1 SDK workflow scenarios exercise lost responses, source-close boundaries, destination collisions and signal admission. | +| [#56](https://github.com/chaptersix/temporal/pull/56) | `service/frontend/migration_delete_race_test.go` includes the native control and a skipped held-create/delete counterexample. V1 termination is mocked. | +| [#57](https://github.com/chaptersix/temporal/pull/57) | `migration_chain_ownership_test.go` adapts owned-descendant, foreign-chain and expired-first-run ownership checks to this baseline, with the persisted request ID and first-run ownership check. Receipt and start-pending persistence are not imported because they make a pre-commit Unavailable permanent. | + +## Driver changes + +- Start calls can be accepted externally, followed by a callback, a lost response, + or successful response delivery. The real handler then records its result. + Tests run engine operations only while all controlled RPCs are blocked. +- External acceptance has stable run identity and survives response loss. Local + observations retain recorded identities separately. Assertions catch accepted + work disappearing, identity mismatches, missing counts and duplicate accounting. +- Side-effect dispatch and progress checks consult `ValidateSideEffectTask`. + Idle-route checks inspect serialized logical tasks and run the production idle + validator. They do not infer progress from stale physical envelopes alone. +- Unexpected patch errors fail the property test. Only the documented closed + schedule rejection is accepted. +- The broad known-bug exemption is removed. Paused completion and catchup expiry + both exposed a discarded selection leaving the remaining buffer deferred. + A narrowly scoped rearming fix now reconsiders deferred work when a discard + leaves no running or in-flight start. Each such immediate pass removes work; + it cannot repeat forever without progress. +- `spec_processor_properties_test.go` supplies independent arithmetic expectations + for interval occurrences, capacity accounting, pagination and jitter boundaries. + It does not establish calendar enumeration or timezone correctness. + +## Reproducible evidence + +All commands run from the repository root. Results recorded 2026-11-11 with +Go `go1.27.3 darwin/arm64` and Rapid `v1.3.0`. + +```sh +CGO_ENABLED=0 go test -tags test_dep ./chasm/chasmtest/... ./chasm/lib/scheduler/... ./service/frontend ./service/worker/scheduler -count=1 +CGO_ENABLED=0 make property-test TEST_RACE_FLAG=off +make mutation-test +CGO_ENABLED=0 make lint-code-fast GOLANGCI_LINT_BASE_REV=03ecd1d228d2b8d9d433da55785afc9aeeead918 GOLANGCI_LINT_FIX=false +``` + +The package tests, 5,000 Rapid cases, all four mutation checks and fast lint pass. Mutations +remove the early callback guard, deferred rearming, positive-capacity admission +and the selected-start reservation independently; each produces assertion +failures in its targeted tests. + +`make property-test` is also a dedicated CI job. The unit-test job retries +failures, and each retry draws a new Rapid seed, so a rare violation could be +retried away there. The dedicated job runs more cases with no retries and +uploads the minimized failfiles from `testdata/rapid`. +The original early callback run failed on `accepted-start-lost`, with ActionCount +still zero after external acceptance. The fix preserves exactly one counted start. + +The catchup-expiry failure is retained both as +`TestExpiredSelectionReleasesRemainingBuffer`. The Rapid failfile is not +committed: it replays raw draws, which any generator change invalidates, and +`testdata/rapid` is ignored. The explicit trace is: fixed initial clock +2020-01-01 UTC; 10-second interval; end after 31 seconds; BUFFER_ALL; 60-second +catchup; 10-minute idle; 5-minute workflow-duration bound; `step, step, step, pure`. +The seeded replay is: + +```sh +CGO_ENABLED=0 go test -tags test_dep ./chasm/lib/scheduler/schedulertest -run TestSchedulerProperties -rapid.seed=7487548259739501542 -count=1 +``` + +## Production fixes made for reproduced bugs + +Each fix has a default-run regression; none is gated. + +- **Backfill cursor replay.** A migrated backfiller carries the V1 exclusive + cursor as its watermark. Watermark presence, not value, marks recorded + progress, so an epoch cursor is honoured. + `TestMigrationBackfill_MigratedResumesAfterWatermark`. +- **Backfill before callback reconciliation.** Migrated backfillers get their + first task only after `SchedulerCallbacksTask` reconciles imported running + state. `TestMigrationBackfill_WaitsForCallbackReconciliation`. +- **Fresh backfill rollback boundary.** Rollback shifts an unstarted range back + 1ms to keep its inclusive first point on V1's exclusive cursor. + `TestMigrationScenario_RollbackFreshBackfill`. +- **Rollback execution phases.** `MigrateToWorkflow` and the migrate task return + the retryable `ErrMigrationActionsPending` while a selected start is + unrecorded or cancel/terminate requests are queued, because V1 state cannot + represent them. `TestRollbackPhaseGate`, `TestRollbackPhaseTransferable`. +- **Rollback destination ownership.** `WorkflowMigrationState.request_id` is + persisted and sent on every start attempt. An already-started destination is + accepted only if its start request, or the create request of its chain's first + run, matches. A foreign chain or a missing first run leaves the source open. + A migration persisted without an ID is assigned one before its first attempt; + if an earlier binary already created its destination under a random ID, the + task fails closed and needs an operator. + `TestMigrationRollbackDestinationCollision`, `TestMigrationRollbackChainOwnership`. +- **Selected-start reservation.** A selected but unacknowledged start holds the + overlap slot. Deferred starts are reconsidered when a start records its run or + is dropped, so a failed start cannot strand the buffer and TERMINATE_OTHER and + CANCEL_OTHER still act. `TestSelectedStartReservesOverlapSlot`, + `TestFailedSelectedStartReleasesOverlapSlot`, + `TestSelectedStartReservationStillTerminatesOther`. +- **Action budget shared across phases.** `InvokerExecuteTask` passed its + per-task context by value, so the terminate, cancel and start phases each + spent `MaxActionsPerExecution` separately. It is now passed by pointer. Found + by the widened property. `TestActionBudgetSharedAcrossPhases`. + +## Harness structure + +The component-agnostic stepping loop is `chasmtest.TaskRunner`: it finds the +next pure or side-effect task, validates and dispatches it, advances the clock, +and exposes hooks for component-specific behaviour. The engine, the runner and +the driver take `chasmtest.TB`, which `*rapid.T` satisfies, so a harness failure +inside a property shrinks like any other. + +`schedulertest` holds the one copy of the scheduler's test scaffolding, used by +both the unit tests in `chasm/lib/scheduler` and the lifecycle driver: + +- fixtures and library wiring (`DefaultSchedule`, `DefaultConfig`, + `NewTestLibrary`); +- `SchedulerEngine`, a `chasmtest.Engine` with typed `Read` and `Update`; +- `StateInvariant`, the stuck and idle-deadline checks, installed on every + engine through `chasmtest.WithInvariantCheck`. The idle-deadline check runs + armed idle tasks through the handler's `Validate`. + +The `Driver` embeds `SchedulerEngine` and adds the fake frontend and history +services, the runner hooks, the schedule operations, and `Snapshot` for the +invariants that need history or the start ledger. It keeps its own `no-stuck` +rule next to the shared one: the test engine never dispatches the framework +visibility task, which stays in the tree and hides a taskless schedule from the +shared count, while the shared rule has no exemption for paused or backfilling +schedules. + +`TestSchedulerProperties` runs as a Rapid state machine (`rt.Repeat`) over +stepping, completion, pause, unpause, trigger, backfill, update, failed starts +and lost start responses, across all six overlap policies. Invariants run in the +after-step hook, so they also see the settled states inside a compound action. +Idle time is drawn; the other tweakables stay at their defaults. Rollback +migration is not a property operation. + +## Unresolved counterexamples and limits + +Four migration cases remain evidence, not fixes. Each is an unconditional +`t.Skip` naming the missing handoff protocol; remove the skip to see the +assertion fail. + +- `TestMigrationCounterexample_TriggerDuringSnapshot`: a signal accepted while + the migrate activity is pending is dropped with the old snapshot. +- `TestMigrationCounterexample_ForwardDestinationCollision`: the forward + activity accepts an unrelated existing CHASM schedule as its destination. +- `TestMigrationCounterexample_LostCreateResponse`: after an ambiguous create, + the V1 source keeps executing transferred work. +- `TestMigrationCounterexample_DeleteDuringCreate`: a create held across + `DeleteSchedule` commits an active scheduler afterwards. + +All four need a handoff protocol that fences source admission and identifies the +migration at the destination; no local production change closes them. The +lost-response case also asserts behaviour at existing workflow versions, so its +fix must be version-gated and its test adjusted. None of these tests establishes +namespace failover safety. + +The default driver remains a bounded, single-schedule environment. Request IDs +are unique per generated occurrence. The external ledger retains deduplication +identities for the whole scenario, assuming no retention expiry. Recorded-result +observation uses buffered starts and recent history: keep each ALLOW_ALL batch +within the retained-history window and observe after every task. Large-batch +history eviction needs a separate observer. Long-running capture/memory work +from [#68](https://github.com/chaptersix/temporal/pull/68) and +[#69](https://github.com/chaptersix/temporal/pull/69) remains a scaling follow-up. + +Serialized restart, physical timer redelivery, durable commits, and History +admission fencing remain separate integration coverage. No new in-memory helper +is presented as persistence or crash evidence. The source-close fault wrapper +only tests transaction outcome/response ordering in the component engine. + +The package tests above and `make property-test` at 5,000 cases also run under +the race detector with no data race reported. +On a macOS host whose default SDK is newer than its linker, the race build fails +with `unknown architecture arm64e.x1`; set `SDKROOT` to the SDK matching the +installed Command Line Tools. diff --git a/go.mod b/go.mod index a013ef821bb..fca0a4545b2 100644 --- a/go.mod +++ b/go.mod @@ -93,6 +93,7 @@ require ( gopkg.in/validator.v2 v2.0.1 gopkg.in/yaml.v3 v3.0.1 modernc.org/sqlite v1.51.0 + pgregory.net/rapid v1.3.0 ) require ( diff --git a/go.sum b/go.sum index 03801765900..af9ae6c5003 100644 --- a/go.sum +++ b/go.sum @@ -699,6 +699,8 @@ modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +pgregory.net/rapid v1.3.0 h1:vBvO0VSqti75J1jjYqpgPNBLKMd1+gxa9fYo7vk/Exc= +pgregory.net/rapid v1.3.0/go.mod h1:dPlE4OBBxgXPqkP79flB6sJL1dx5azpI7HQ9MY9Z7uk= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= diff --git a/service/frontend/admin_handler.go b/service/frontend/admin_handler.go index 20ee940bc6d..c76634d2d62 100644 --- a/service/frontend/admin_handler.go +++ b/service/frontend/admin_handler.go @@ -2543,6 +2543,11 @@ func (adh *AdminHandler) migrateScheduleToWorkflow( request *adminservice.MigrateScheduleRequest, namespaceID string, ) (*adminservice.MigrateScheduleResponse, error) { + // The scheduler stores this ID and uses it to start the V1 workflow. + migrationRequestID := request.GetRequestId() + if migrationRequestID == "" { + migrationRequestID = uuid.NewString() + } workflowID := scheduler.WorkflowIDPrefix + request.GetScheduleId() descResp, err := adh.historyClient.DescribeWorkflowExecution(ctx, &historyservice.DescribeWorkflowExecutionRequest{ NamespaceId: namespaceID, @@ -2587,7 +2592,7 @@ func (adh *AdminHandler) migrateScheduleToWorkflow( NamespaceId: namespaceID, ScheduleId: request.GetScheduleId(), Identity: request.GetIdentity(), - RequestId: request.GetRequestId(), + RequestId: migrationRequestID, }, ) if err != nil { diff --git a/service/frontend/admin_handler_test.go b/service/frontend/admin_handler_test.go index 929f4e11da1..a69491c736f 100644 --- a/service/frontend/admin_handler_test.go +++ b/service/frontend/admin_handler_test.go @@ -2377,6 +2377,29 @@ func (f *fakeSchedulerClient) MigrateToWorkflow(ctx context.Context, req *schedu return f.migrateToWorkflowFn(ctx, req) } +func (s *adminHandlerSuite) TestMigrateScheduleToWorkflowGeneratesRequestID() { + s.mockNamespaceCache.EXPECT().GetNamespaceID(s.namespace).Return(s.namespaceID, nil) + s.mockHistoryClient.EXPECT().DescribeWorkflowExecution(gomock.Any(), gomock.Any()).Return( + nil, serviceerror.NewNotFound("workflow not found")) + + var capturedReq *schedulerpb.MigrateToWorkflowRequest + s.handler.schedulerClient = &fakeSchedulerClient{ + migrateToWorkflowFn: func(_ context.Context, req *schedulerpb.MigrateToWorkflowRequest) (*schedulerpb.MigrateToWorkflowResponse, error) { + capturedReq = req + return &schedulerpb.MigrateToWorkflowResponse{}, nil + }, + } + + _, err := s.handler.MigrateSchedule(context.Background(), &adminservice.MigrateScheduleRequest{ + Namespace: s.namespace.String(), + ScheduleId: "test-schedule", + Target: adminservice.MigrateScheduleRequest_SCHEDULER_TARGET_WORKFLOW, + Identity: "test-identity", + }) + s.NoError(err) + s.NoError(uuid.Validate(capturedReq.GetRequestId())) +} + func (s *adminHandlerSuite) TestMigrateScheduleToWorkflow() { s.mockNamespaceCache.EXPECT().GetNamespaceID(s.namespace).Return(s.namespaceID, nil) s.mockHistoryClient.EXPECT().DescribeWorkflowExecution(gomock.Any(), &historyservice.DescribeWorkflowExecutionRequest{ diff --git a/service/frontend/migration_delete_race_test.go b/service/frontend/migration_delete_race_test.go new file mode 100644 index 00000000000..80ce9250d57 --- /dev/null +++ b/service/frontend/migration_delete_race_test.go @@ -0,0 +1,106 @@ +package frontend + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + schedulepb "go.temporal.io/api/schedule/v1" + "go.temporal.io/api/serviceerror" + "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/server/api/historyservice/v1" + "go.temporal.io/server/api/historyservicemock/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/chasmtest" + chasmscheduler "go.temporal.io/server/chasm/lib/scheduler" + schedulerpb "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" + "go.temporal.io/server/common/clock" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/namespace" + legacyscheduler "go.temporal.io/server/service/worker/scheduler" + "go.uber.org/mock/gomock" + "google.golang.org/grpc" + "google.golang.org/protobuf/types/known/timestamppb" +) + +type migrationDeleteRaceClient struct { + schedulerpb.SchedulerServiceClient + engineCtx context.Context + ref chasm.ComponentRef +} + +func (c *migrationDeleteRaceClient) DeleteSchedule(_ context.Context, req *schedulerpb.DeleteScheduleRequest, _ ...grpc.CallOption) (*schedulerpb.DeleteScheduleResponse, error) { + response, _, err := chasm.UpdateComponent(c.engineCtx, c.ref, func(s *chasmscheduler.Scheduler, ctx chasm.MutableContext, _ struct{}) (*schedulerpb.DeleteScheduleResponse, error) { + return s.Delete(ctx, req) + }, struct{}{}) + return response, err +} + +func TestMigrationDeleteNativeControl(t *testing.T) { testMigrationDeleteRace(t, false) } +func TestMigrationCounterexample_DeleteDuringCreate(t *testing.T) { + t.Skip("A create held across DeleteSchedule commits an active scheduler afterwards; this needs a handoff protocol that records the delete as a tombstone the in-flight migration create must observe before committing.") + testMigrationDeleteRace(t, true) +} + +func testMigrationDeleteRace(t *testing.T, inFlight bool) { + t.Helper() + logger := log.NewNoopLogger() + config := &chasmscheduler.Config{Tweakables: func(string) chasmscheduler.Tweakables { return chasmscheduler.DefaultTweakables }} + builder := legacyscheduler.NewSpecBuilder(func() int { return 0 }, func() int { return 0 }) + processor := chasmscheduler.NewSpecProcessor(config, metrics.NoopMetricsHandler, logger, builder) + registry := chasm.NewRegistry(logger) + require.NoError(t, registry.Register(&chasm.CoreLibrary{})) + require.NoError(t, registry.Register(chasmscheduler.NewLibrary(config, nil, + chasmscheduler.NewSchedulerIdleTaskHandler(chasmscheduler.SchedulerIdleTaskHandlerOptions{Config: config, MetricsHandler: metrics.NoopMetricsHandler, BaseLogger: logger}), nil, + chasmscheduler.NewGeneratorTaskHandler(chasmscheduler.GeneratorTaskHandlerOptions{Config: config, MetricsHandler: metrics.NoopMetricsHandler, BaseLogger: logger, SpecBuilder: builder, SpecProcessor: processor}), nil, nil, nil, nil))) + now := time.Date(2026, 9, 4, 12, 0, 0, 0, time.UTC) + engine := chasmtest.NewEngine(t, registry, chasmtest.WithTimeSource(clock.NewEventTimeSource().Update(now))) + engineCtx := chasm.NewEngineContext(context.Background(), engine) + key := chasm.ExecutionKey{NamespaceID: "ns-id", BusinessID: "schedule-id"} + ref := chasm.NewComponentRef[*chasmscheduler.Scheduler](key) + create := func() error { + _, err := chasm.StartExecution(engineCtx, key, chasmscheduler.CreateSchedulerFromMigration, &schedulerpb.CreateFromMigrationStateRequest{NamespaceId: key.NamespaceID, State: &schedulerpb.SchedulerMigrationState{ + SchedulerState: &schedulerpb.SchedulerState{Namespace: "ns", NamespaceId: key.NamespaceID, ScheduleId: key.BusinessID, Schedule: &schedulepb.Schedule{Spec: &schedulepb.ScheduleSpec{}, State: &schedulepb.ScheduleState{Paused: true}}, Info: &schedulepb.ScheduleInfo{}}, + GeneratorState: &schedulerpb.GeneratorState{LastProcessedTime: timestamppb.New(now)}, InvokerState: &schedulerpb.InvokerState{}, + }}) + return err + } + started, release, created := make(chan struct{}), make(chan struct{}), make(chan error, 1) + var releaseOnce sync.Once + releaseCreate := func() { releaseOnce.Do(func() { close(release) }) } + if inFlight { + // Unblock the create goroutine even if the test fails early. + t.Cleanup(releaseCreate) + go func() { close(started); <-release; created <- create() }() + <-started + } else { + require.NoError(t, create()) + } + ctrl := gomock.NewController(t) + namespaces := namespace.NewMockRegistry(ctrl) + namespaces.EXPECT().GetNamespaceID(namespace.Name("ns")).Return(namespace.ID("ns-id"), nil).Times(2) + history := historyservicemock.NewMockHistoryServiceClient(ctrl) + history.EXPECT().TerminateWorkflowExecution(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, req *historyservice.TerminateWorkflowExecutionRequest, _ ...grpc.CallOption) (*historyservice.TerminateWorkflowExecutionResponse, error) { + require.Equal(t, legacyscheduler.WorkflowIDPrefix+key.BusinessID, req.TerminateRequest.WorkflowExecution.WorkflowId) + return &historyservice.TerminateWorkflowExecutionResponse{}, nil + }).Times(1) + handler := &WorkflowHandler{logger: logger, namespaceRegistry: namespaces, historyClient: history, schedulerClient: &migrationDeleteRaceClient{engineCtx: engineCtx, ref: ref}, config: &Config{EnableSchedules: func(string) bool { return true }, EnableCHASMSchedulerCreation: func(string) bool { return true }}} + _, deleteErr := handler.DeleteSchedule(context.Background(), &workflowservice.DeleteScheduleRequest{Namespace: "ns", ScheduleId: key.BusinessID}) + if inFlight { + releaseCreate() + require.NoError(t, <-created) + } + require.NoError(t, deleteErr) + _, err := chasm.ReadComponent(engineCtx, ref, func(s *chasmscheduler.Scheduler, _ chasm.Context, _ struct{}) (struct{}, error) { + require.True(t, s.Closed, "successful DeleteSchedule must not be followed by an active migrated scheduler") + return struct{}{}, nil + }, struct{}{}) + if err != nil { + _, ok := errors.AsType[*serviceerror.NotFound](err) + require.True(t, ok, "unexpected error: %v", err) + } +} diff --git a/service/worker/scheduler/migration_scenarios_test.go b/service/worker/scheduler/migration_scenarios_test.go new file mode 100644 index 00000000000..0a6a64ec63f --- /dev/null +++ b/service/worker/scheduler/migration_scenarios_test.go @@ -0,0 +1,260 @@ +package scheduler + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + schedulepb "go.temporal.io/api/schedule/v1" + "go.temporal.io/api/serviceerror" + taskqueuepb "go.temporal.io/api/taskqueue/v1" + workflowpb "go.temporal.io/api/workflow/v1" + "go.temporal.io/sdk/testsuite" + "go.temporal.io/sdk/workflow" + schedulespb "go.temporal.io/server/api/schedule/v1" + schedulerpb "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" + "go.temporal.io/server/chasm/lib/scheduler/migration" + "go.temporal.io/server/common" + "google.golang.org/grpc" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func newMigrationScenario(t *testing.T) (*testsuite.TestWorkflowEnvironment, *schedulespb.StartScheduleArgs) { + t.Helper() + var suite testsuite.WorkflowTestSuite + env := suite.NewTestWorkflowEnvironment() + env.SetStartTime(baseStartTime) + env.RegisterDelayedCallback(func() { env.SignalWorkflow(SignalNameForceCAN, nil) }, 3*time.Hour) + t.Cleanup(func() { env.AssertExpectations(t) }) + return env, &schedulespb.StartScheduleArgs{ + Schedule: &schedulepb.Schedule{ + Spec: &schedulepb.ScheduleSpec{Interval: []*schedulepb.IntervalSpec{{Interval: durationpb.New(time.Hour)}}}, + Action: &schedulepb.ScheduleAction{Action: &schedulepb.ScheduleAction_StartWorkflow{ + StartWorkflow: &workflowpb.NewWorkflowExecutionInfo{ + WorkflowId: "action", WorkflowType: &commonpb.WorkflowType{Name: "action"}, TaskQueue: &taskqueuepb.TaskQueue{Name: "queue"}, + }, + }}, + Policies: &schedulepb.SchedulePolicies{OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL}, + State: &schedulepb.ScheduleState{}, + }, + Info: &schedulepb.ScheduleInfo{}, + State: &schedulespb.InternalState{Namespace: "ns", NamespaceId: testNamespaceID, ScheduleId: "schedule", ConflictToken: InitialConflictToken}, + } +} + +func TestMigrationScenario_TriggerBeforeSnapshot(t *testing.T) { + checkMigrationSignalAccounting(t, false) +} + +func TestMigrationCounterexample_TriggerDuringSnapshot(t *testing.T) { + t.Skip("A trigger accepted while the migrate activity is pending is lost with the stale snapshot; this needs a handoff protocol that fences source signal admission once the snapshot is taken or forwards late signals to the destination.") + checkMigrationSignalAccounting(t, true) +} + +func checkMigrationSignalAccounting(t *testing.T, duringMigration bool) { + t.Helper() + env, args := newMigrationScenario(t) + enabled := duringMigration + var destination *schedulerpb.SchedulerMigrationState + starts := 0 + env.OnActivity(new(activities).StartWorkflow, mock.Anything, mock.Anything).Maybe().Return( + func(context.Context, *schedulespb.StartWorkflowRequest) (*schedulespb.StartWorkflowResponse, error) { + starts++ + return &schedulespb.StartWorkflowResponse{RunId: "run", RealStartTime: timestamppb.New(env.Now())}, nil + }) + env.OnActivity(new(activities).MigrateScheduleToChasm, mock.Anything, mock.Anything).Once().After(2 * time.Second).Return( + func(_ context.Context, req *schedulerpb.CreateFromMigrationStateRequest) error { + destination = common.CloneProto(req.State) + return nil + }) + delivered := false + env.RegisterDelayedCallback(func() { + require.False(t, env.IsWorkflowCompleted()) + env.SignalWorkflow(SignalNamePatch, &schedulepb.SchedulePatch{ + TriggerImmediately: &schedulepb.TriggerImmediatelyRequest{OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL}, + }) + delivered = true + }, time.Second) + if !duringMigration { + env.RegisterDelayedCallback(func() { + enabled = true + env.SignalWorkflow(SignalNameMigrateToChasm, nil) + }, 3*time.Second) + } + env.ExecuteWorkflow(func(ctx workflow.Context, input *schedulespb.StartScheduleArgs) error { + return schedulerWorkflowWithSpecBuilder(ctx, input, newSpecBuilderForTest(0, 0), schedulerDynamicConfig{ + enableCHASMMigration: func() bool { return enabled }, migrateWithRunningWorkflows: func() bool { return true }, + versionOverride: func() int { return int(MigrationHandoffFixes) }, + }) + }, args) + require.NoError(t, env.GetWorkflowError()) + require.True(t, delivered) + require.NotNil(t, destination) + pending := 0 + for _, start := range destination.GetInvokerState().GetBufferedStarts() { + if start.Manual && start.RunId == "" && start.Completed == nil { + pending++ + } + } + for _, backfill := range destination.Backfillers { + if backfill.GetTriggerRequest() != nil { + pending++ + } + } + require.Equal(t, 1, starts+pending, "a delivered trigger must execute or remain actionable at the destination") +} + +type migrationCommitClient struct { + schedulerpb.SchedulerServiceClient + destination *schedulerpb.SchedulerMigrationState + commit bool + calls int +} + +func (c *migrationCommitClient) CreateFromMigrationState( + _ context.Context, req *schedulerpb.CreateFromMigrationStateRequest, _ ...grpc.CallOption, +) (*schedulerpb.CreateFromMigrationStateResponse, error) { + c.calls++ + if c.calls == 1 { + if c.commit { + c.destination = common.CloneProto(req.State) + } + return nil, serviceerror.NewUnavailable("injected response loss") + } + if c.destination != nil { + return nil, serviceerror.NewAlreadyExists("destination committed on first attempt") + } + c.destination = common.CloneProto(req.State) + return &schedulerpb.CreateFromMigrationStateResponse{}, nil +} + +func TestMigrationScenario_FailureBeforeCommit(t *testing.T) { + checkMigrationCommitAccounting(t, false, false, MigrationHandoffFixes) +} + +func TestMigrationScenario_NativeBackfillIncludesStart(t *testing.T) { + checkRollbackBackfillStart(t, false) +} + +func TestMigrationScenario_RollbackFreshBackfill(t *testing.T) { + checkRollbackBackfillStart(t, true) +} + +func TestMigrationCounterexample_ForwardDestinationCollision(t *testing.T) { + t.Skip("The forward activity treats AlreadyExists as success for an unrelated CHASM schedule; this needs a migration identity on the destination so the source can tell its own earlier create from a foreign schedule.") + env, args := newMigrationScenario(t) + when := timestamppb.New(baseStartTime) + args.State.BufferedStarts = []*schedulespb.BufferedStart{{ + NominalTime: when, ActualTime: when, Manual: true, + OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL, + }} + client := &mockSchedulerClient{migrateErr: serviceerror.NewAlreadyExists("unrelated CHASM schedule owns this ID")} + a := newTestActivities(client, testNamespaceID) + env.OnActivity(new(activities).MigrateScheduleToChasm, mock.Anything, mock.Anything).Return( + func(ctx context.Context, req *schedulerpb.CreateFromMigrationStateRequest) error { + return a.MigrateScheduleToChasm(ctx, req) + }) + env.ExecuteWorkflow(func(ctx workflow.Context, input *schedulespb.StartScheduleArgs) error { + return schedulerWorkflowWithSpecBuilder(ctx, input, newSpecBuilderForTest(0, 0), schedulerDynamicConfig{ + enableCHASMMigration: func() bool { return true }, migrateWithRunningWorkflows: func() bool { return true }, + }) + }, args) + require.True(t, workflow.IsContinueAsNewError(env.GetWorkflowError()), + "the source must remain active when the destination ID belongs to an unrelated schedule") +} + +func checkRollbackBackfillStart(t *testing.T, rollback bool) { + t.Helper() + env, args := newMigrationScenario(t) + args.Schedule.State.Paused = true + env.RegisterDelayedCallback(func() { env.SignalWorkflow(SignalNameForceCAN, nil) }, time.Second) + when := timestamppb.New(baseStartTime.Add(-time.Hour)) + backfill := &schedulepb.BackfillRequest{ + StartTime: when, EndTime: when, OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL, + } + if rollback { + args = migration.CHASMToLegacyStartScheduleArgs(&schedulerpb.SchedulerState{ + Schedule: args.Schedule, Info: args.Info, Namespace: args.State.Namespace, + NamespaceId: args.State.NamespaceId, ScheduleId: args.State.ScheduleId, + }, &schedulerpb.GeneratorState{LastProcessedTime: timestamppb.New(baseStartTime)}, &schedulerpb.InvokerState{}, + map[string]*schedulerpb.BackfillerState{"fresh": { + Request: &schedulerpb.BackfillerState_BackfillRequest{BackfillRequest: backfill}, + }}, nil, nil, nil, baseStartTime) + } else { + args.InitialPatch = &schedulepb.SchedulePatch{BackfillRequest: []*schedulepb.BackfillRequest{backfill}} + } + starts := 0 + env.OnActivity(new(activities).StartWorkflow, mock.Anything, mock.Anything).Maybe().Return( + func(context.Context, *schedulespb.StartWorkflowRequest) (*schedulespb.StartWorkflowResponse, error) { + starts++ + return &schedulespb.StartWorkflowResponse{RunId: "backfilled-run", RealStartTime: timestamppb.New(env.Now())}, nil + }) + env.ExecuteWorkflow(SchedulerWorkflow, args) + require.True(t, workflow.IsContinueAsNewError(env.GetWorkflowError())) + require.Equal(t, 1, starts, "the inclusive start of a fresh CHASM backfill must remain actionable after rollback") +} + +func TestMigrationCounterexample_LostCreateResponse(t *testing.T) { + t.Skip("After a lost create response the V1 source keeps executing work the committed destination also owns; this needs a handoff protocol that fences source execution until the destination commit is confirmed, gated on a new workflow version.") + for _, version := range []SchedulerWorkflowVersion{TriggerImmediatelyTimestamp, MigrationHandoffFixes} { + for _, disable := range []bool{false, true} { + t.Run(fmt.Sprintf("version_%d_disable_%t", version, disable), func(t *testing.T) { + checkMigrationCommitAccounting(t, true, disable, version) + }) + } + } +} + +func checkMigrationCommitAccounting(t *testing.T, commit, disable bool, version SchedulerWorkflowVersion) { + t.Helper() + env, args := newMigrationScenario(t) + when := timestamppb.New(baseStartTime) + args.State.BufferedStarts = []*schedulespb.BufferedStart{{ + NominalTime: when, ActualTime: when, Manual: true, OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL, + }} + client := &migrationCommitClient{commit: commit} + a := newTestActivities(client, testNamespaceID) + enabled := true + a.migrationEnabled = func() bool { return enabled } + env.OnActivity(new(activities).MigrateScheduleToChasm, mock.Anything, mock.Anything).Return( + func(ctx context.Context, req *schedulerpb.CreateFromMigrationStateRequest) error { + err := a.MigrateScheduleToChasm(ctx, req) + if disable { + enabled = false + } + return err + }) + var sourceRequests []*schedulespb.StartWorkflowRequest + env.OnActivity(new(activities).StartWorkflow, mock.Anything, mock.Anything).Return( + func(_ context.Context, req *schedulespb.StartWorkflowRequest) (*schedulespb.StartWorkflowResponse, error) { + sourceRequests = append(sourceRequests, common.CloneProto(req)) + return &schedulespb.StartWorkflowResponse{RunId: fmt.Sprintf("run-%d", len(sourceRequests)), RealStartTime: timestamppb.New(env.Now())}, nil + }) + env.ExecuteWorkflow(func(ctx workflow.Context, input *schedulespb.StartScheduleArgs) error { + return schedulerWorkflowWithSpecBuilder(ctx, input, newSpecBuilderForTest(0, 0), schedulerDynamicConfig{ + enableCHASMMigration: func() bool { return enabled }, migrateWithRunningWorkflows: func() bool { return true }, + versionOverride: func() int { return int(version) }, versionCeiling: func() int { return int(version) }, + }) + }, args) + if disable { + require.True(t, workflow.IsContinueAsNewError(env.GetWorkflowError())) + } else { + require.NoError(t, env.GetWorkflowError()) + require.Equal(t, 2, client.calls) + } + require.NotEmpty(t, sourceRequests) + require.NotNil(t, client.destination) + manualAtDestination := 0 + for _, start := range client.destination.GetInvokerState().GetBufferedStarts() { + if start.Manual && start.RunId == "" && start.Completed == nil { + manualAtDestination++ + } + } + require.Zero(t, manualAtDestination, "source executed the manual action after migration returned an error; the committed destination must not still own it") +}