diff --git a/.github/workflows/property-test-soak.yml b/.github/workflows/property-test-soak.yml new file mode 100644 index 00000000000..a234d590d9b --- /dev/null +++ b/.github/workflows/property-test-soak.yml @@ -0,0 +1,74 @@ +name: Property Test Soak + +# The per-PR Property test job runs 1000 checks so it stays cheap +# enough to gate on. This workflow is where the input space actually gets +# explored: the same tests, same single attempt, but orders of magnitude more +# generated cases, on a fresh random seed each night. +# +# It is deliberately NOT a required check. A soak finding is a real bug, but it +# is found on an input no PR introduced, so it should not block unrelated merges. +# When it fails, download the counterexample artifact to reproduce the case +# locally, then pin it as a hand-written deterministic test. The fail files are +# gitignored rather than committed, because they replay a raw draw sequence and +# stop reproducing silently once a generator changes. + +on: + schedule: + # Runs at 0600 UTC daily (night in the US), before the other daily jobs. + - cron: "0 6 * * *" + workflow_dispatch: # Allow manual trigger + inputs: + rapid_checks: + description: Number of generated cases per property + required: false + default: "100000" + +permissions: + contents: read + +concurrency: + group: property-test-soak-${{ github.ref }} + cancel-in-progress: true + +jobs: + soak: + name: Property test soak + runs-on: ubuntu-24.04-arm + env: + RAPID_CHECKS: ${{ inputs.rapid_checks || '100000' }} + TEST_TIMEOUT: 50m + GITHUB_TIMEOUT: 60 + # Single attempt, as in the per-PR job: a retry would re-draw the seed and + # discard the counterexample the soak exists to find. + MAX_TEST_ATTEMPTS: 1 + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Prepare Go + uses: ./.github/actions/prepare-go + + - name: Run property test soak + timeout-minutes: ${{ fromJSON(env.GITHUB_TIMEOUT) }} + run: | + echo "::notice::Soaking with RAPID_CHECKS=$RAPID_CHECKS" + export TEST_RUNNER_TIMEOUT="$(("$GITHUB_TIMEOUT" - 1))m" + ./develop/github/monitor_test.sh make property-test-coverage + + - name: Upload property test counterexamples + if: failure() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: property-test-soak-counterexamples + path: "**/testdata/rapid/**/*.fail" + if-no-files-found: ignore + + - name: Post-test reporting + if: always() + uses: ./.github/actions/post-test-reporting + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + job_name: Property test soak + artifact_name_suffix: property-test-soak diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index ca8dc7fabf9..455366d3b82 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -324,6 +324,54 @@ jobs: path: ${{ env.LEAK_OUTPUT_DIR }} if-no-files-found: ignore + property-test: + name: Property test + needs: [pre-build] + runs-on: ubuntu-24.04-arm + env: + TEST_TIMEOUT: 5m + GITHUB_TIMEOUT: 10 + # Property tests run a single attempt: a retry would re-draw rapid's random + # seed and replace a real counterexample with a different, likely passing, + # input set. The Makefile locks the attempt count, and this keeps + # post-test-reporting's "fewer junit files than attempts means crash" + # heuristic correct for a one-attempt job. + MAX_TEST_ATTEMPTS: 1 + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + ref: ${{ env.COMMIT }} + + - name: Prepare Go + uses: ./.github/actions/prepare-go + + - name: Run property tests + timeout-minutes: ${{ fromJSON(env.GITHUB_TIMEOUT) }} + run: | + export TEST_RUNNER_TIMEOUT="$(("$GITHUB_TIMEOUT" - 1))m" + ./develop/github/monitor_test.sh make property-test-coverage + + # rapid writes a minimized reproducer next to the failing package. Without + # this it is discarded with the runner and the counterexample is lost. + # Commit a downloaded .fail file to replay it on every later run. + - name: Upload property test counterexamples + if: failure() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: property-test-counterexamples + path: "**/testdata/rapid/**/*.fail" + if-no-files-found: ignore + + - name: Post-test reporting + if: always() + uses: ./.github/actions/post-test-reporting + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + job_name: Property test + artifact_name_suffix: property-test + integration-test: name: Integration test needs: [pre-build] @@ -494,6 +542,7 @@ jobs: needs: - misc-checks - unit-test + - property-test - integration-test - functional-test runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index f6e1955edc7..48b2dda4a29 100644 --- a/.gitignore +++ b/.gitignore @@ -48,3 +48,9 @@ # Ignoring AI agent files .agents/ + +# rapid failfiles replay raw draw sequences, so any change to a generator makes a +# stored file decode to different values and stop reproducing — silently, with no +# error. Pin regressions with hand-written tests instead. CI uploads failfiles as +# artifacts on failure for local reproduction. +**/testdata/rapid/ diff --git a/Makefile b/Makefile index 6c881c70323..f85e88cb389 100644 --- a/Makefile +++ b/Makefile @@ -127,9 +127,17 @@ MIXED_BRAIN_TEST_ROOT := ./tests/mixedbrain DB_INTEGRATION_TEST_ROOT := ./common/persistence/tests DB_TOOL_INTEGRATION_TEST_ROOT := ./tools/tests INTEGRATION_TEST_DIRS := $(DB_INTEGRATION_TEST_ROOT) $(DB_TOOL_INTEGRATION_TEST_ROOT) ./temporaltest +# Property tests run in their own tier so they can be run without retries; see +# the Property tests section below for why. The tier is selected by directory +# name: any package directory ending in "proptest" joins it, and is filtered out +# of UNIT_TEST_DIRS below so it does not also run in the retrying unit tier, +# where a counterexample would be masked by a retry. Deriving the list from a +# naming convention means a new property test package cannot be silently left in +# the unit tier by forgetting to register it here. +PROPERTY_TEST_DIRS := $(patsubst %/,%,$(filter %proptest/,$(TEST_DIRS))) TESTCORE_UNITTESTS := ./tests/testcore ifeq ($(UNIT_TEST_DIRS),) -UNIT_TEST_DIRS := $(filter-out $(FUNCTIONAL_TEST_ROOT)% $(FUNCTIONAL_TEST_XDC_ROOT)% $(FUNCTIONAL_TEST_NDC_ROOT)% $(MIXED_BRAIN_TEST_ROOT)% $(DB_INTEGRATION_TEST_ROOT)% $(DB_TOOL_INTEGRATION_TEST_ROOT)% ./temporaltest%,$(TEST_DIRS)) +UNIT_TEST_DIRS := $(filter-out $(FUNCTIONAL_TEST_ROOT)% $(FUNCTIONAL_TEST_XDC_ROOT)% $(FUNCTIONAL_TEST_NDC_ROOT)% $(MIXED_BRAIN_TEST_ROOT)% $(DB_INTEGRATION_TEST_ROOT)% $(DB_TOOL_INTEGRATION_TEST_ROOT)% ./temporaltest% $(addsuffix %,$(PROPERTY_TEST_DIRS)),$(TEST_DIRS)) # Testcore unit tests are filtered out by the FUNCTIONAL_TEST_ROOT pattern, need to add them back manually. UNIT_TEST_DIRS += $(TESTCORE_UNITTESTS) @@ -481,6 +489,37 @@ integration-test: clean-test-output @CGO_ENABLED=$(TEST_CGO_ENABLED) go test $(INTEGRATION_TEST_DIRS) $(COMPILED_TEST_ARGS) 2>&1 | tee -a test.log @$(MAKE) verify-test-log +# Property tests check invariants against randomly-generated inputs, so they need +# a different runner configuration than ordinary unit tests: +# +# 1. They must NOT be retried. test-runner exits with the exit code of its last +# attempt only, and a retry re-runs the test in a fresh process where rapid +# draws a new random seed. A real counterexample would therefore be replaced +# by a different (likely passing) input set and the job would go green. The +# attempt count is locked with := so that the MAX_TEST_ATTEMPTS environment +# variable set in the CI workflow cannot re-enable retries here. +# 2. The number of generated cases is a knob. The default matches rapid's own +# default and costs ~60ms; the nightly soak workflow raises it a long way. +# +# When a property fails, rapid writes a minimized reproducer to +# testdata/rapid//--.fail. Use it to reproduce the +# failure locally, then pin the case as a hand-written deterministic test. The +# fail files themselves are gitignored: they replay a raw draw sequence, so any +# change to a generator makes them decode to different values and stop +# reproducing the bug silently. +# +# See docs/development/testing.md ("Property tests") for how to reproduce a +# counterexample, when to commit one, and how to add a package to this tier. +PROPERTY_TEST_MAX_ATTEMPTS := 1 +# Generated cases per property. Above rapid's own default of 100, to catch more +# before a merge; the nightly soak workflow raises it much further. +RAPID_CHECKS ?= 1000 + +property-test: clean-test-output + @printf $(COLOR) "Run property tests ($(RAPID_CHECKS) checks)..." + @RAPID_CHECKS=$(RAPID_CHECKS) CGO_ENABLED=$(TEST_CGO_ENABLED) go test $(PROPERTY_TEST_DIRS) $(COMPILED_TEST_ARGS) 2>&1 | tee -a test.log + @$(MAKE) verify-test-log + functional-test: clean-test-output @printf $(COLOR) "Run functional tests..." @CGO_ENABLED=$(TEST_CGO_ENABLED) go test $(FUNCTIONAL_TEST_ROOT) $(COMPILED_TEST_ARGS) -persistenceType=$(PERSISTENCE_TYPE) -persistenceDriver=$(PERSISTENCE_DRIVER) 2>&1 | tee -a test.log @@ -521,7 +560,7 @@ verify-test-log: @grep -q "^ok" test.log || (echo "TEST FAILURE: no passing test found in test.log" && exit 1) @! grep -q "^--- FAIL" test.log || (echo "TEST FAILURE: failing test found in test.log" && exit 1) -test: unit-test integration-test functional-test +test: unit-test property-test integration-test functional-test ##### Coverage & Reporting ##### $(TEST_OUTPUT_ROOT): @@ -544,6 +583,13 @@ integration-test-coverage: prepare-coverage-test CGO_ENABLED=$(TEST_CGO_ENABLED) go run ./cmd/tools/test-runner test --gotestsum-path=$(GOTESTSUM) --max-attempts=$(MAX_TEST_ATTEMPTS) $(TEST_RUNNER_TIMEOUT_ARG) --junitfile=$(NEW_REPORT) -- \ $(COMPILED_TEST_ARGS) -coverprofile=$(NEW_COVER_PROFILE) $(INTEGRATION_TEST_DIRS) +# Runs through test-runner (not plain go test) to keep the JUnit report, coverage +# and flaky-test reporting, but with retries disabled. See PROPERTY_TEST_DIRS. +property-test-coverage: prepare-coverage-test + @printf $(COLOR) "Run property tests with coverage ($(RAPID_CHECKS) checks)..." + RAPID_CHECKS=$(RAPID_CHECKS) CGO_ENABLED=$(TEST_CGO_ENABLED) go run ./cmd/tools/test-runner test --gotestsum-path=$(GOTESTSUM) --max-attempts=$(PROPERTY_TEST_MAX_ATTEMPTS) $(TEST_RUNNER_TIMEOUT_ARG) --junitfile=$(NEW_REPORT) -- \ + $(COMPILED_TEST_ARGS) -coverprofile=$(NEW_COVER_PROFILE) $(PROPERTY_TEST_DIRS) + # MUST use the same build flags as functional-test-coverage and functional-test-{xdc,ndc}-coverage for best build caching. pre-build-functional-test-coverage: prepare-coverage-test CGO_ENABLED=$(TEST_CGO_ENABLED) go test -c -cover -o /dev/null $(COMPILED_TEST_ARGS) $(COVERPKG_FLAG) $(FUNCTIONAL_TEST_ROOT) diff --git a/chasm/chasmtest/test_engine.go b/chasm/chasmtest/test_engine.go index 8b33b40c3cf..aaa1d1d61b7 100644 --- a/chasm/chasmtest/test_engine.go +++ b/chasm/chasmtest/test_engine.go @@ -6,7 +6,6 @@ import ( "fmt" "maps" "sync" - "testing" "time" enumspb "go.temporal.io/api/enums/v1" @@ -25,19 +24,30 @@ import ( ) type ( + // TB is the subset of [testing.TB] the engine, gomock controllers and + // testlogger need. Both *testing.T and *rapid.T satisfy it, so an engine can + // be built inside a rapid property and have failures flow through rapid's + // shrinking. It is declared here, rather than imported from rapid, so this + // package does not depend on rapid. + TB interface { + testlogger.CleanupCapableT + Errorf(format string, args ...any) + Context() context.Context + } + EngineOption func(*Engine) // InvariantCheck validates an execution after a successful transition. The // node is clean, so checks may inspect its snapshot without affecting the // state being validated. - InvariantCheck func(*testing.T, *chasm.Node, chasm.RootComponent) + InvariantCheck func(TB, *chasm.Node, chasm.RootComponent) // Engine is a lightweight in memory CHASM engine for unit tests. It implements // [chasm.Engine] and supports the full set of conflict and reuse policies, as // well as blocking [PollComponent] with [NotifyExecution], matching the behavior // of the production engine as closely as possible without persistence or shard logic. Engine struct { - t *testing.T + t TB registry *chasm.Registry logger log.Logger metrics metrics.Handler @@ -107,7 +117,7 @@ var defaultTransitionOptions = chasm.TransitionOptions{ var _ chasm.Engine = (*Engine)(nil) func NewEngine( - t *testing.T, + t TB, registry *chasm.Registry, opts ...EngineOption, ) *Engine { diff --git a/chasm/chasmtest/test_engine_test.go b/chasm/chasmtest/test_engine_test.go index 48dc3c54708..390595d2ef7 100644 --- a/chasm/chasmtest/test_engine_test.go +++ b/chasm/chasmtest/test_engine_test.go @@ -50,7 +50,7 @@ func TestTasksArePhysicallyGenerated(t *testing.T) { func TestInvariantCheckRunsAfterSuccessfulTransitions(t *testing.T) { var calls []string check := func(name string) chasmtest.InvariantCheck { - return func(checkT *testing.T, node *chasm.Node, root chasm.RootComponent) { + return func(checkT chasmtest.TB, node *chasm.Node, root chasm.RootComponent) { checkT.Helper() require.Same(checkT, t, checkT) require.IsType(checkT, &tests.PayloadStore{}, root) @@ -122,7 +122,7 @@ func TestInvariantCheckRunsAfterSuccessfulTransitions(t *testing.T) { // sub-components its update adds must become nodes in the tree, not only exist in memory. func TestUpdateWithStartSyncsSubComponentsAddedByUpdate(t *testing.T) { var nodePaths []string - check := func(_ *testing.T, node *chasm.Node, _ chasm.RootComponent) { + check := func(_ chasmtest.TB, node *chasm.Node, _ chasm.RootComponent) { nodePaths = slices.Collect(maps.Keys(node.Snapshot(nil).Nodes)) } e, _ := startStoreWithEngineOptions(t, 0, []chasmtest.EngineOption{chasmtest.WithInvariantCheck(check)}) @@ -368,7 +368,7 @@ func TestRecordsCallerPrincipal(t *testing.T) { worker := &commonpb.Principal{Type: "jwt", Name: "worker"} var rootAttributes *persistencespb.ChasmComponentAttributes - captureRoot := func(_ *testing.T, node *chasm.Node, _ chasm.RootComponent) { + captureRoot := func(_ chasmtest.TB, node *chasm.Node, _ chasm.RootComponent) { rootAttributes = node.Snapshot(nil).Nodes[""].GetMetadata().GetComponentAttributes() } diff --git a/chasm/lib/scheduler/generator_wakeup_test.go b/chasm/lib/scheduler/generator_wakeup_test.go index c895cbb0a12..b35cbd75f03 100644 --- a/chasm/lib/scheduler/generator_wakeup_test.go +++ b/chasm/lib/scheduler/generator_wakeup_test.go @@ -54,7 +54,7 @@ func TestGeneratorNextWakeupTimeStaysAtPendingJitteredFire(t *testing.T) { require.NoError(t, err) var timerDeadlines []time.Time observeTimers := func(config *engineTestConfig) { - config.engineOpts = append(config.engineOpts, chasmtest.WithInvariantCheck(func(_ *testing.T, node *chasm.Node, _ chasm.RootComponent) { + config.engineOpts = append(config.engineOpts, chasmtest.WithInvariantCheck(func(_ chasmtest.TB, node *chasm.Node, _ chasm.RootComponent) { timerDeadlines = nil generator := node.Snapshot(nil).Nodes[generatorPath] for _, task := range generator.GetMetadata().GetComponentAttributes().GetPureTasks() { diff --git a/chasm/lib/scheduler/helper_test.go b/chasm/lib/scheduler/helper_test.go index b73ed03e28f..295a431d761 100644 --- a/chasm/lib/scheduler/helper_test.go +++ b/chasm/lib/scheduler/helper_test.go @@ -278,7 +278,7 @@ func newTestEngineContextFromConfig( require.NoError(t, registry.Register(newTestLibrary(logger, specProcessor))) config.engineOpts = append(config.engineOpts, chasmtest.WithInvariantCheck( - func(t *testing.T, node *chasm.Node, root chasm.RootComponent) { + func(t chasmtest.TB, node *chasm.Node, root chasm.RootComponent) { requireValidSchedulerState(t, registry, node, root) }, )) diff --git a/chasm/lib/scheduler/invariant_test.go b/chasm/lib/scheduler/invariant_test.go index b35b08be0f2..699eb70cc95 100644 --- a/chasm/lib/scheduler/invariant_test.go +++ b/chasm/lib/scheduler/invariant_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/require" "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/chasmtest" "go.temporal.io/server/chasm/lib/scheduler" "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" "google.golang.org/protobuf/types/known/timestamppb" @@ -139,7 +140,7 @@ func idleTaskCount(registry *chasm.Registry, node *chasm.Node) int { // Only asserted in this direction. Sentinels legitimately arm an idle task // without setting IdleCloseTime (see NewSentinel), so an armed task with no // declared deadline is not a defect. -func requireIdleCloseTimeBacked(t *testing.T, registry *chasm.Registry, node *chasm.Node, sched *scheduler.Scheduler) { +func requireIdleCloseTimeBacked(t chasmtest.TB, registry *chasm.Registry, node *chasm.Node, sched *scheduler.Scheduler) { t.Helper() if sched.GetSentinel() || sched.GetClosed() || sched.GetIdleCloseTime() == nil { @@ -153,7 +154,7 @@ func requireIdleCloseTimeBacked(t *testing.T, registry *chasm.Registry, node *ch } // requireNotStuck asserts the stuckness invariant against a clean tree. -func requireNotStuck(t *testing.T, node *chasm.Node, sched *scheduler.Scheduler) { +func requireNotStuck(t chasmtest.TB, node *chasm.Node, sched *scheduler.Scheduler) { t.Helper() total, byPath := liveTaskCount(node) @@ -162,7 +163,7 @@ func requireNotStuck(t *testing.T, node *chasm.Node, sched *scheduler.Scheduler) } func requireValidSchedulerState( - t *testing.T, + t chasmtest.TB, registry *chasm.Registry, node *chasm.Node, root chasm.RootComponent, diff --git a/chasm/lib/scheduler/schedulertest/property_test.go b/chasm/lib/scheduler/schedulerproptest/property_test.go similarity index 80% rename from chasm/lib/scheduler/schedulertest/property_test.go rename to chasm/lib/scheduler/schedulerproptest/property_test.go index 02f1bf53133..62913ff7069 100644 --- a/chasm/lib/scheduler/schedulertest/property_test.go +++ b/chasm/lib/scheduler/schedulerproptest/property_test.go @@ -1,4 +1,13 @@ -package schedulertest_test +// Package schedulerproptest holds the property tests for the CHASM scheduler, +// driving the [schedulertest] harness against randomly-generated schedules and +// operation sequences. +// +// Property tests live in their own directory rather than alongside the unit +// tests in package schedulertest because the CI tier that runs them is selected +// by directory name (any directory matching *proptest) and runs without retries +// — retrying a property test re-draws rapid's seed and discards the +// counterexample. See docs/development/testing.md ("Property tests"). +package schedulerproptest import ( "testing" @@ -19,9 +28,10 @@ import ( // failing operation sequence to a minimal reproducer — directly useful for the // "stuck open" bug class. // -// The outer *testing.T owns the engine scaffolding (it requires a real -// testing.TB); invariant violations are reported through the *rapid.T so rapid -// can shrink them. +// The engine scaffolding is built on the *rapid.T rather than the outer +// *testing.T, so a harness failure (a gomock expectation, an engine assertion, +// an error-level log) shrinks and is attributed to the generated case that +// caused it, exactly like an invariant violation. func TestSchedulerProperties(t *testing.T) { rapid.Check(t, func(rt *rapid.T) { start := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) @@ -30,10 +40,10 @@ func TestSchedulerProperties(t *testing.T) { idle := rapid.SampledFrom([]time.Duration{10 * time.Minute, time.Hour, 2 * time.Hour}).Draw(rt, "idleTime") sched := drawSchedule(rt, start, idle) - d := schedulertest.NewDriver(t, + d := schedulertest.NewDriver(rt, schedulertest.WithStartTime(start), schedulertest.WithSchedule(sched), - schedulertest.WithConfig(configWith(func(tw *scheduler.Tweakables) { + schedulertest.WithConfig(schedulertest.ConfigWith(func(tw *scheduler.Tweakables) { tw.IdleTime = idle })), // InvokerExecuteTask may lag the timers, and workflows may run, but @@ -42,7 +52,7 @@ func TestSchedulerProperties(t *testing.T) { schedulertest.WithMaxSideEffectLag(idle/2), schedulertest.WithMaxRunDuration(idle/2), ) - hook := checkInvariantsIgnoringKnownBugs(rt) + hook := schedulertest.CheckInvariantsHook(rt) d.AfterStep = hook // Check the initial settled state too. @@ -132,24 +142,3 @@ func drawSchedule(rt *rapid.T, start time.Time, idle time.Duration) *schedulepb. return s } - -// checkInvariantsIgnoringKnownBugs is CheckInvariantsHook minus one known bug -// (TestLedger_DeferredStartsStrandedByPausedCompletion): a lost start where -// every pending start is deferred and nothing is running to re-enable them. -// Delete this once that bug is fixed. -func checkInvariantsIgnoringKnownBugs(rt *rapid.T) func(*schedulertest.Driver) { - var prev *schedulertest.Snapshot - return func(d *schedulertest.Driver) { - rt.Helper() - cur := d.Snapshot() - for _, v := range schedulertest.CheckInvariants(prev, cur) { - if v.Name == "ledger-lost-start" && cur.RunningWorkflows == 0 && - cur.DeferredStarts == len(cur.PendingStarts) { - continue - } - rt.Fatalf("scheduler invariant violation at step (now=%s): %s", - cur.Now.Format("2006-01-02T15:04:05"), v) - } - prev = &cur - } -} diff --git a/chasm/lib/scheduler/schedulertest/driver.go b/chasm/lib/scheduler/schedulertest/driver.go index 93e8b94cb89..a2c04a50e27 100644 --- a/chasm/lib/scheduler/schedulertest/driver.go +++ b/chasm/lib/scheduler/schedulertest/driver.go @@ -6,7 +6,6 @@ import ( "fmt" "maps" "sync" - "testing" "time" commonpb "go.temporal.io/api/common/v1" @@ -52,7 +51,7 @@ var maxKeyFireTime = tasks.MaximumKey.FireTime // re-arms a generator task), so callers bound the run with maxSteps; quiescence is // reached only for paused/exhausted/closed schedules. type Driver struct { - T *testing.T + T chasmtest.TB Engine *chasmtest.Engine Registry *chasm.Registry TimeSource *clock.EventTimeSource @@ -153,7 +152,7 @@ func WithMaxRunDuration(d time.Duration) DriverOption { // NewDriver constructs a Driver and starts a scheduler execution via the // production CreateScheduler path. startTime defaults to a fixed instant for // determinism. -func NewDriver(t *testing.T, opts ...DriverOption) *Driver { +func NewDriver(t chasmtest.TB, opts ...DriverOption) *Driver { t.Helper() d, cfg := newDriverScaffold(t, opts...) @@ -175,7 +174,7 @@ func NewDriver(t *testing.T, opts ...DriverOption) *Driver { // NewDriverFromMigration constructs a Driver and starts a scheduler execution // from a migrated V1 state via the production CreateSchedulerFromMigration path. // This is the V1->V2 import side of a migration round trip. -func NewDriverFromMigration(t *testing.T, req *schedulerpb.CreateFromMigrationStateRequest, opts ...DriverOption) *Driver { +func NewDriverFromMigration(t chasmtest.TB, req *schedulerpb.CreateFromMigrationStateRequest, opts ...DriverOption) *Driver { t.Helper() d, _ := newDriverScaffold(t, opts...) d.startExecution(func(mctx chasm.MutableContext) (*scheduler.Scheduler, error) { @@ -186,7 +185,7 @@ func NewDriverFromMigration(t *testing.T, req *schedulerpb.CreateFromMigrationSt // newDriverScaffold builds the engine, registry, clients, and time source, but // does not start an execution. -func newDriverScaffold(t *testing.T, opts ...DriverOption) (*Driver, *driverConfig) { +func newDriverScaffold(t chasmtest.TB, opts ...DriverOption) (*Driver, *driverConfig) { t.Helper() cfg := &driverConfig{ diff --git a/chasm/lib/scheduler/schedulertest/ledger_test.go b/chasm/lib/scheduler/schedulertest/ledger_test.go index b930f99f84d..a8e8f5f802f 100644 --- a/chasm/lib/scheduler/schedulertest/ledger_test.go +++ b/chasm/lib/scheduler/schedulertest/ledger_test.go @@ -13,15 +13,6 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" ) -// configWith returns DefaultConfig with Tweakables adjusted by fn. -func configWith(fn func(*scheduler.Tweakables)) *scheduler.Config { - cfg := schedulertest.DefaultConfig() - tweakables := scheduler.DefaultTweakables - fn(&tweakables) - cfg.Tweakables = func(string) scheduler.Tweakables { return tweakables } - return cfg -} - // oneShotSchedule fires exactly once, at start+offset. func oneShotSchedule(start time.Time, offset time.Duration) *schedulepb.Schedule { s := schedulertest.DefaultSchedule() @@ -76,7 +67,7 @@ func TestLedger_FinalStartPendingAtIdle(t *testing.T) { d := schedulertest.NewDriver(t, schedulertest.WithStartTime(start), schedulertest.WithSchedule(oneShotSchedule(start, tc.offset)), - schedulertest.WithConfig(configWith(func(tw *scheduler.Tweakables) { + schedulertest.WithConfig(schedulertest.ConfigWith(func(tw *scheduler.Tweakables) { tw.IdleTime = idle tw.EnablePendingStartIdleFix = tc.fixOn })), @@ -169,14 +160,18 @@ func uniq(names []string) []string { return out } -// A start deferred behind a running workflow is only re-enabled by a workflow -// completion. If that completion arrives while the schedule is paused, -// ProcessBuffer drops the first start (paused) and re-defers the rest with -// nothing running, so no completion will ever re-enable them. Unpause doesn't -// re-process the buffer either; the starts (including a manual trigger, which -// pause allows) sit until the idle task closes the schedule. +// A start deferred behind a running workflow is normally re-enabled by that +// workflow completing. When the completion arrives while the schedule is +// paused, ProcessBuffer selects the first start, drops it because the schedule +// is paused, and re-defers the rest — leaving deferred starts with nothing +// running whose completion could ever wake them. +// +// Invoker.addTasks closes that gap: with no pending or running start it calls +// reenableDeferredStarts, which returns the deferred starts to Attempt == 0 so +// the next ProcessBuffer pass reconsiders them. This test guards that path, in +// particular that a manual trigger (which pause permits) still starts rather +// than sitting in the buffer until the idle task closes the schedule. func TestLedger_DeferredStartsStrandedByPausedCompletion(t *testing.T) { - t.Skip("known bug: deferred starts stranded after a completion while paused") const idle = time.Hour start := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) @@ -188,7 +183,7 @@ func TestLedger_DeferredStartsStrandedByPausedCompletion(t *testing.T) { d := schedulertest.NewDriver(t, schedulertest.WithStartTime(start), schedulertest.WithSchedule(s), - schedulertest.WithConfig(configWith(func(tw *scheduler.Tweakables) { tw.IdleTime = idle })), + schedulertest.WithConfig(schedulertest.ConfigWith(func(tw *scheduler.Tweakables) { tw.IdleTime = idle })), schedulertest.WithAfterStep(collectViolations(&violations)), ) diff --git a/chasm/lib/scheduler/schedulertest/library.go b/chasm/lib/scheduler/schedulertest/library.go index e8b827e321b..70c36954832 100644 --- a/chasm/lib/scheduler/schedulertest/library.go +++ b/chasm/lib/scheduler/schedulertest/library.go @@ -89,6 +89,15 @@ func DefaultConfig() *scheduler.Config { } } +// ConfigWith returns [DefaultConfig] with Tweakables adjusted by fn. +func ConfigWith(fn func(*scheduler.Tweakables)) *scheduler.Config { + cfg := DefaultConfig() + tweakables := scheduler.DefaultTweakables + fn(&tweakables) + cfg.Tweakables = func(string) scheduler.Tweakables { return tweakables } + return cfg +} + // NewRealSpecProcessor builds a real SpecProcessor with no-op metrics, matching // production spec processing so generated buffered starts and next-times are // realistic. diff --git a/cmd/tools/test/find_altered_tests.go b/cmd/tools/test/find_altered_tests.go index 600f5f9870b..0d5cee42104 100644 --- a/cmd/tools/test/find_altered_tests.go +++ b/cmd/tools/test/find_altered_tests.go @@ -33,10 +33,12 @@ import ( // FUNCTIONAL_TEST_XDC_ROOT // FUNCTIONAL_TEST_NDC_ROOT // INTEGRATION_TEST_DIRS +// PROPERTY_TEST_DIRS // UNIT_TEST_DIRS var CategoryDirs = map[string][]string{ "unit": {"./client", "./common", "./internal", "./service", "./temporal", "./tools", "./cmd"}, "integration": {"./common/persistence/tests", "./tools/tests", "./temporaltest"}, + "property": {"./chasm/lib/scheduler/schedulerproptest"}, "functional": {"./tests"}, "functional_ndc": {"./tests/ndc"}, "functional_xdc": {"./tests/xdc"}, diff --git a/docs/development/testing.md b/docs/development/testing.md index b087bffee52..1163d3557e4 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -87,6 +87,143 @@ unless there is a reason not to. `make parallelize-tests` can be used to automatically add `t.Parallel()`. Use `//parallelize:ignore` to opt your test out of it. +## Property tests + +Property tests assert invariants against randomly-generated inputs instead of hand-written cases. +They use [rapid](https://pkg.go.dev/pgregory.net/rapid), which generates the inputs and, on +failure, shrinks them to a minimal counterexample. + +They run in their own tier, separate from unit tests, because they need a different runner +configuration; see [Why retries are disabled](#why-retries-are-disabled). + +The tier is every package directory whose name ends in `proptest`; see +[Adding a property test](#adding-a-property-test). + +### Running locally + +```bash +make property-test # 1000 generated cases per property +make property-test RAPID_CHECKS=50000 # soak before pushing a change to a covered package +``` + +Or with plain `go test`. Note that `-rapid.*` are test *binary* flags, so the package has to come +before them — otherwise `go test` stops parsing and fails with `[setup failed]`: + +```bash +go test -tags test_dep ./chasm/lib/scheduler/schedulerproptest/ -run TestSchedulerProperties -rapid.checks=1000 +``` + +### Configuration + +`RAPID_CHECKS` sets the number of generated cases per property, and defaults to `1000` — cheap +enough to run on every PR. Every rapid flag also has an environment variable form +(`-rapid.checks` / `RAPID_CHECKS`, and so on); see rapid's documentation for the full set. + +Seeds are **not** pinned in CI. Retries are disabled instead, which keeps a counterexample from +being discarded while still letting coverage accumulate across commits. + +### In CI + +| Job | Workflow | Gates merges | +| --- | --- | --- | +| `Property test` | `run-tests.yml`, every PR | intended | +| `Property test soak` | `property-test-soak.yml`, nightly 0600 UTC | no | + +Both run a single attempt. The soak also accepts a manual `workflow_dispatch` with a `rapid_checks` +input. + +`Property test` only blocks a merge once it is listed as a required status check in the repository's +branch protection settings; the workflow alone does not make it one. + +The soak is deliberately never a required check. A counterexample it finds is a real bug, but it is +found on an input that no particular PR introduced, so it must not block unrelated merges. + +### When a counterexample is found + +rapid prints the minimized counterexample along with a line naming both ways to reproduce it — the +*fail file* it just wrote, and the seed: + +``` +To reproduce, specify -run="" -rapid.failfile="" (or -rapid.seed=) +``` + +Either form works. The fail file path is relative to the package directory: + +```bash +go test -tags test_dep ./chasm/lib/scheduler/schedulerproptest/ -run TestSchedulerProperties \ + -rapid.failfile="testdata/rapid/TestSchedulerProperties/TestSchedulerProperties--.fail" +``` + +In CI the fail file is uploaded as a job artifact — `property-test-counterexamples`, or +`property-test-soak-counterexamples` for the nightly. Download it to reproduce locally. + +### Pinning a counterexample + +**Pin the counterexample as a hand-written deterministic test, in the same change as the fix.** +Translate the shrunk inputs into an explicit test — `TestLedger_DeferredStartsStrandedByPausedCompletion` +in [schedulertest](../../chasm/lib/scheduler/schedulertest) is an example — and put it next to the +other unit tests for the package, not in the property tier. + +**Do not commit the fail file.** `testdata/rapid/` is gitignored, because a fail file is not a +durable regression test: it replays a raw sequence of draws, so adding, removing or reordering a +draw in the generator makes the stored sequence decode to different values. It then stops +reproducing the bug **silently** — rapid reports no error, the replay simply passes, and the +protection is gone without anything failing to tell you. A hand-written test says what it means and +breaks loudly when behaviour changes. + +Use the fail file for what it is good at — reproducing the failure *now*, while you write the +real test. + +### Deferring a fix + +If the bug cannot be fixed immediately, the thing to avoid is any marker that passes silently once +the bug is gone. A `t.Skip` and a filter in the invariant hook both do, which is how this +repository ended up carrying a skip *and* an invariant exemption for a stranding bug that had +already been fixed on main seven hours earlier — the skip's own comment asked for its deletion, and +nothing deleted it because nothing broke when the bug disappeared. + +So, in order of preference: + +- Fix it. A shrunk counterexample is usually small, and the fix often is too. +- Open an issue, and paste in both the seed and the fail file's contents — a few lines of hex that + fully describe the case. Do not rely on the CI artifact; those expire. +- If the property has to keep running while the bug is open, constrain the **generator** rather + than filtering violations in the hook. A generator constraint is visible in the inputs and + self-documenting; a violation filter silently swallows every other bug of the same shape, not + just the one it names. +- If you must leave a marker, make it fail when the bug is fixed — for example a deterministic test + that asserts the current wrong outcome behind a `// BUG(#NNNNN):` comment, so whoever fixes the + bug is forced to update it. Put the issue number in any `t.Skip` message so it is at least + greppable. + +### Why retries are disabled + +`property-test-coverage` passes `--max-attempts=1` to the test runner, unlike the unit and +functional tiers, which retry up to `MAX_TEST_ATTEMPTS` times. **Do not re-enable retries for this +tier.** + +The test runner exits with the exit code of its *last* attempt, and each retry re-runs the test in a +fresh process, where rapid draws a new random seed. A retried property test therefore checks a +different set of inputs than the one that failed: the counterexample would be replaced by a +likely-passing run, and the job would go green having found a real bug. + +### Adding a property test + +**Put property tests in their own package directory, named with a `proptest` suffix.** The tier is +selected by directory name — `PROPERTY_TEST_DIRS` in the [Makefile](../../Makefile) matches any +directory ending in `proptest` — so creating one enrolls it automatically and excludes it from the +unit tier. Nothing needs registering. Also add it to the `property` category in +[find_altered_tests.go](../../cmd/tools/test/find_altered_tests.go), which is a separate list. + +The dedicated directory is not stylistic. The tier subtracts whole directories from +`UNIT_TEST_DIRS`, so a property test placed inside the package it exercises forces a choice between +pulling that package's entire unit test suite into the no-retry tier, or leaving the property test +in the retrying tier where its counterexamples are discarded. + +Because a sibling directory can only reach exported API, the package under test needs an exported +harness — see [schedulertest](../../chasm/lib/scheduler/schedulertest), which is a non-test package +precisely so property tests can import its driver. + ## Test helpers Test helpers can be found in the [common/testing](../../common/testing) package.