From 8504c1abf90df92202a29cbd681a154d3d0e74fd Mon Sep 17 00:00:00 2001 From: chaptersix <13949480+chaptersix@users.noreply.github.com> Date: Fri, 13 Nov 2026 23:02:27 -0600 Subject: [PATCH 1/5] Run CHASM scheduler property tests in a dedicated no-retry CI tier The scheduler property test landed inside schedulertest, which the Makefile globs into UNIT_TEST_DIRS. That tier runs through test-runner with --max-attempts=3, and test-runner exits with the exit code of its last attempt only. Each retry re-runs the test in a fresh process where rapid draws a new random seed, so a retried property test checks a different set of inputs than the one that failed: a real counterexample is replaced by a likely-passing run and the job goes green having found a bug. Give property tests their own tier instead: - PROPERTY_TEST_DIRS is derived from a directory naming convention (any package directory ending in "proptest"), so a new property test package is enrolled, and excluded from the retrying unit tier, automatically rather than by a hand-maintained list whose failure mode is silent. - property-test and property-test-coverage run the tier. The latter goes through test-runner to keep the JUnit report, coverage and flaky-test reporting, but with --max-attempts=1. PROPERTY_TEST_MAX_ATTEMPTS is assigned with := so the MAX_TEST_ATTEMPTS environment variable set in the CI workflow cannot re-enable retries for this tier. - A Property test job runs the tier on every PR at rapid's default 100 checks and uploads rapid's minimized counterexample as an artifact on failure. - A nightly Property Test Soak workflow runs the same tier at 100k checks. It is deliberately not a required check: a counterexample it finds is a real bug, but on an input that no particular PR introduced. Move the property test to chasm/lib/scheduler/schedulerproptest, and promote the configWith helper it shared with ledger_test.go to schedulertest.ConfigWith. A dedicated directory is required because the tier subtracts whole directories from UNIT_TEST_DIRS; schedulertest returns to the unit tier, where retries are appropriate for its unit tests. Document the tier in docs/development/testing.md, including when and where to commit a counterexample: rapid replays every fail file under testdata/rapid// ahead of the random cases, so one belongs with the fix that resolves it and must not be committed for an unfixed bug, which would fail every PR. Co-Authored-By: Claude Opus 5 --- .github/workflows/property-test-soak.yml | 74 +++++++++++ .github/workflows/run-tests.yml | 48 +++++++ Makefile | 46 ++++++- .../property_test.go | 13 +- .../scheduler/schedulertest/ledger_test.go | 13 +- chasm/lib/scheduler/schedulertest/library.go | 9 ++ cmd/tools/test/find_altered_tests.go | 2 + docs/development/testing.md | 119 ++++++++++++++++++ 8 files changed, 309 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/property-test-soak.yml rename chasm/lib/scheduler/{schedulertest => schedulerproptest}/property_test.go (89%) diff --git a/.github/workflows/property-test-soak.yml b/.github/workflows/property-test-soak.yml new file mode 100644 index 00000000000..47f696fd76b --- /dev/null +++ b/.github/workflows/property-test-soak.yml @@ -0,0 +1,74 @@ +name: Property Test Soak + +# The per-PR Property test job runs rapid's default 100 checks so it stays cheap +# enough to gate on. This workflow is where the input space actually gets +# explored: the same tests, same single attempt, but orders of magnitude more +# generated cases, on a fresh random seed each night. +# +# It is deliberately NOT a required check. A soak finding is a real bug, but it +# is found on an input no PR introduced, so it should not block unrelated merges. +# When it fails, download the counterexample artifact and commit the .fail file +# under the package's testdata/rapid/ directory: rapid replays every such file +# ahead of the random cases, which turns the finding into a permanent regression +# case that the per-PR job then enforces. + +on: + schedule: + # Runs at 0600 UTC daily (night in the US), before the other daily jobs. + - cron: "0 6 * * *" + workflow_dispatch: # Allow manual trigger + inputs: + rapid_checks: + description: Number of generated cases per property + required: false + default: "100000" + +permissions: + contents: read + +concurrency: + group: property-test-soak-${{ github.ref }} + cancel-in-progress: true + +jobs: + soak: + name: Property test soak + runs-on: ubuntu-24.04-arm + env: + RAPID_CHECKS: ${{ inputs.rapid_checks || '100000' }} + TEST_TIMEOUT: 50m + GITHUB_TIMEOUT: 60 + # Single attempt, as in the per-PR job: a retry would re-draw the seed and + # discard the counterexample the soak exists to find. + MAX_TEST_ATTEMPTS: 1 + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Prepare Go + uses: ./.github/actions/prepare-go + + - name: Run property test soak + timeout-minutes: ${{ fromJSON(env.GITHUB_TIMEOUT) }} + run: | + echo "::notice::Soaking with RAPID_CHECKS=$RAPID_CHECKS" + export TEST_RUNNER_TIMEOUT="$(("$GITHUB_TIMEOUT" - 1))m" + ./develop/github/monitor_test.sh make property-test-coverage + + - name: Upload property test counterexamples + if: failure() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: property-test-soak-counterexamples + path: "**/testdata/rapid/**" + if-no-files-found: ignore + + - name: Post-test reporting + if: always() + uses: ./.github/actions/post-test-reporting + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + job_name: Property test soak + artifact_name_suffix: property-test-soak diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index ca8dc7fabf9..0eaef0e9666 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -324,6 +324,54 @@ jobs: path: ${{ env.LEAK_OUTPUT_DIR }} if-no-files-found: ignore + property-test: + name: Property test + needs: [pre-build] + runs-on: ubuntu-24.04-arm + env: + TEST_TIMEOUT: 5m + GITHUB_TIMEOUT: 10 + # Property tests run a single attempt: a retry would re-draw rapid's random + # seed and replace a real counterexample with a different, likely passing, + # input set. The Makefile locks the attempt count, and this keeps + # post-test-reporting's "fewer junit files than attempts means crash" + # heuristic correct for a one-attempt job. + MAX_TEST_ATTEMPTS: 1 + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + ref: ${{ env.COMMIT }} + + - name: Prepare Go + uses: ./.github/actions/prepare-go + + - name: Run property tests + timeout-minutes: ${{ fromJSON(env.GITHUB_TIMEOUT) }} + run: | + export TEST_RUNNER_TIMEOUT="$(("$GITHUB_TIMEOUT" - 1))m" + ./develop/github/monitor_test.sh make property-test-coverage + + # rapid writes a minimized reproducer next to the failing package. Without + # this it is discarded with the runner and the counterexample is lost. + # Commit a downloaded .fail file to replay it on every later run. + - name: Upload property test counterexamples + if: failure() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: property-test-counterexamples + path: "**/testdata/rapid/**" + if-no-files-found: ignore + + - name: Post-test reporting + if: always() + uses: ./.github/actions/post-test-reporting + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + job_name: Property test + artifact_name_suffix: property-test + integration-test: name: Integration test needs: [pre-build] diff --git a/Makefile b/Makefile index 6c881c70323..430e4dc974f 100644 --- a/Makefile +++ b/Makefile @@ -127,9 +127,17 @@ MIXED_BRAIN_TEST_ROOT := ./tests/mixedbrain DB_INTEGRATION_TEST_ROOT := ./common/persistence/tests DB_TOOL_INTEGRATION_TEST_ROOT := ./tools/tests INTEGRATION_TEST_DIRS := $(DB_INTEGRATION_TEST_ROOT) $(DB_TOOL_INTEGRATION_TEST_ROOT) ./temporaltest +# Property tests run in their own tier so they can be run without retries; see +# the Property tests section below for why. The tier is selected by directory +# name: any package directory ending in "proptest" joins it, and is filtered out +# of UNIT_TEST_DIRS below so it does not also run in the retrying unit tier, +# where a counterexample would be masked by a retry. Deriving the list from a +# naming convention means a new property test package cannot be silently left in +# the unit tier by forgetting to register it here. +PROPERTY_TEST_DIRS := $(patsubst %/,%,$(filter %proptest/,$(TEST_DIRS))) TESTCORE_UNITTESTS := ./tests/testcore ifeq ($(UNIT_TEST_DIRS),) -UNIT_TEST_DIRS := $(filter-out $(FUNCTIONAL_TEST_ROOT)% $(FUNCTIONAL_TEST_XDC_ROOT)% $(FUNCTIONAL_TEST_NDC_ROOT)% $(MIXED_BRAIN_TEST_ROOT)% $(DB_INTEGRATION_TEST_ROOT)% $(DB_TOOL_INTEGRATION_TEST_ROOT)% ./temporaltest%,$(TEST_DIRS)) +UNIT_TEST_DIRS := $(filter-out $(FUNCTIONAL_TEST_ROOT)% $(FUNCTIONAL_TEST_XDC_ROOT)% $(FUNCTIONAL_TEST_NDC_ROOT)% $(MIXED_BRAIN_TEST_ROOT)% $(DB_INTEGRATION_TEST_ROOT)% $(DB_TOOL_INTEGRATION_TEST_ROOT)% ./temporaltest% $(addsuffix %,$(PROPERTY_TEST_DIRS)),$(TEST_DIRS)) # Testcore unit tests are filtered out by the FUNCTIONAL_TEST_ROOT pattern, need to add them back manually. UNIT_TEST_DIRS += $(TESTCORE_UNITTESTS) @@ -481,6 +489,33 @@ integration-test: clean-test-output @CGO_ENABLED=$(TEST_CGO_ENABLED) go test $(INTEGRATION_TEST_DIRS) $(COMPILED_TEST_ARGS) 2>&1 | tee -a test.log @$(MAKE) verify-test-log +# Property tests check invariants against randomly-generated inputs, so they need +# a different runner configuration than ordinary unit tests: +# +# 1. They must NOT be retried. test-runner exits with the exit code of its last +# attempt only, and a retry re-runs the test in a fresh process where rapid +# draws a new random seed. A real counterexample would therefore be replaced +# by a different (likely passing) input set and the job would go green. The +# attempt count is locked with := so that the MAX_TEST_ATTEMPTS environment +# variable set in the CI workflow cannot re-enable retries here. +# 2. The number of generated cases is a knob. The default matches rapid's own +# default and costs ~60ms; the nightly soak workflow raises it a long way. +# +# When a property fails, rapid writes a minimized reproducer to +# testdata/rapid//--.fail and replays every such +# file ahead of the random cases on subsequent runs. Commit those files to turn a +# counterexample into a permanent regression case. +# +# See docs/development/testing.md ("Property tests") for how to reproduce a +# counterexample, when to commit one, and how to add a package to this tier. +PROPERTY_TEST_MAX_ATTEMPTS := 1 +RAPID_CHECKS ?= 100 + +property-test: clean-test-output + @printf $(COLOR) "Run property tests ($(RAPID_CHECKS) checks)..." + @RAPID_CHECKS=$(RAPID_CHECKS) CGO_ENABLED=$(TEST_CGO_ENABLED) go test $(PROPERTY_TEST_DIRS) $(COMPILED_TEST_ARGS) 2>&1 | tee -a test.log + @$(MAKE) verify-test-log + functional-test: clean-test-output @printf $(COLOR) "Run functional tests..." @CGO_ENABLED=$(TEST_CGO_ENABLED) go test $(FUNCTIONAL_TEST_ROOT) $(COMPILED_TEST_ARGS) -persistenceType=$(PERSISTENCE_TYPE) -persistenceDriver=$(PERSISTENCE_DRIVER) 2>&1 | tee -a test.log @@ -521,7 +556,7 @@ verify-test-log: @grep -q "^ok" test.log || (echo "TEST FAILURE: no passing test found in test.log" && exit 1) @! grep -q "^--- FAIL" test.log || (echo "TEST FAILURE: failing test found in test.log" && exit 1) -test: unit-test integration-test functional-test +test: unit-test property-test integration-test functional-test ##### Coverage & Reporting ##### $(TEST_OUTPUT_ROOT): @@ -544,6 +579,13 @@ integration-test-coverage: prepare-coverage-test CGO_ENABLED=$(TEST_CGO_ENABLED) go run ./cmd/tools/test-runner test --gotestsum-path=$(GOTESTSUM) --max-attempts=$(MAX_TEST_ATTEMPTS) $(TEST_RUNNER_TIMEOUT_ARG) --junitfile=$(NEW_REPORT) -- \ $(COMPILED_TEST_ARGS) -coverprofile=$(NEW_COVER_PROFILE) $(INTEGRATION_TEST_DIRS) +# Runs through test-runner (not plain go test) to keep the JUnit report, coverage +# and flaky-test reporting, but with retries disabled. See PROPERTY_TEST_DIRS. +property-test-coverage: prepare-coverage-test + @printf $(COLOR) "Run property tests with coverage ($(RAPID_CHECKS) checks)..." + RAPID_CHECKS=$(RAPID_CHECKS) CGO_ENABLED=$(TEST_CGO_ENABLED) go run ./cmd/tools/test-runner test --gotestsum-path=$(GOTESTSUM) --max-attempts=$(PROPERTY_TEST_MAX_ATTEMPTS) $(TEST_RUNNER_TIMEOUT_ARG) --junitfile=$(NEW_REPORT) -- \ + $(COMPILED_TEST_ARGS) -coverprofile=$(NEW_COVER_PROFILE) $(PROPERTY_TEST_DIRS) + # MUST use the same build flags as functional-test-coverage and functional-test-{xdc,ndc}-coverage for best build caching. pre-build-functional-test-coverage: prepare-coverage-test CGO_ENABLED=$(TEST_CGO_ENABLED) go test -c -cover -o /dev/null $(COMPILED_TEST_ARGS) $(COVERPKG_FLAG) $(FUNCTIONAL_TEST_ROOT) diff --git a/chasm/lib/scheduler/schedulertest/property_test.go b/chasm/lib/scheduler/schedulerproptest/property_test.go similarity index 89% rename from chasm/lib/scheduler/schedulertest/property_test.go rename to chasm/lib/scheduler/schedulerproptest/property_test.go index 02f1bf53133..fb0a252e332 100644 --- a/chasm/lib/scheduler/schedulertest/property_test.go +++ b/chasm/lib/scheduler/schedulerproptest/property_test.go @@ -1,4 +1,13 @@ -package schedulertest_test +// Package schedulerproptest holds the property tests for the CHASM scheduler, +// driving the [schedulertest] harness against randomly-generated schedules and +// operation sequences. +// +// Property tests live in their own directory rather than alongside the unit +// tests in package schedulertest because the CI tier that runs them is selected +// by directory name (any directory matching *proptest) and runs without retries +// — retrying a property test re-draws rapid's seed and discards the +// counterexample. See docs/development/testing.md ("Property tests"). +package schedulerproptest import ( "testing" @@ -33,7 +42,7 @@ func TestSchedulerProperties(t *testing.T) { d := schedulertest.NewDriver(t, schedulertest.WithStartTime(start), schedulertest.WithSchedule(sched), - schedulertest.WithConfig(configWith(func(tw *scheduler.Tweakables) { + schedulertest.WithConfig(schedulertest.ConfigWith(func(tw *scheduler.Tweakables) { tw.IdleTime = idle })), // InvokerExecuteTask may lag the timers, and workflows may run, but diff --git a/chasm/lib/scheduler/schedulertest/ledger_test.go b/chasm/lib/scheduler/schedulertest/ledger_test.go index b930f99f84d..0df8a529124 100644 --- a/chasm/lib/scheduler/schedulertest/ledger_test.go +++ b/chasm/lib/scheduler/schedulertest/ledger_test.go @@ -13,15 +13,6 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" ) -// configWith returns DefaultConfig with Tweakables adjusted by fn. -func configWith(fn func(*scheduler.Tweakables)) *scheduler.Config { - cfg := schedulertest.DefaultConfig() - tweakables := scheduler.DefaultTweakables - fn(&tweakables) - cfg.Tweakables = func(string) scheduler.Tweakables { return tweakables } - return cfg -} - // oneShotSchedule fires exactly once, at start+offset. func oneShotSchedule(start time.Time, offset time.Duration) *schedulepb.Schedule { s := schedulertest.DefaultSchedule() @@ -76,7 +67,7 @@ func TestLedger_FinalStartPendingAtIdle(t *testing.T) { d := schedulertest.NewDriver(t, schedulertest.WithStartTime(start), schedulertest.WithSchedule(oneShotSchedule(start, tc.offset)), - schedulertest.WithConfig(configWith(func(tw *scheduler.Tweakables) { + schedulertest.WithConfig(schedulertest.ConfigWith(func(tw *scheduler.Tweakables) { tw.IdleTime = idle tw.EnablePendingStartIdleFix = tc.fixOn })), @@ -188,7 +179,7 @@ func TestLedger_DeferredStartsStrandedByPausedCompletion(t *testing.T) { d := schedulertest.NewDriver(t, schedulertest.WithStartTime(start), schedulertest.WithSchedule(s), - schedulertest.WithConfig(configWith(func(tw *scheduler.Tweakables) { tw.IdleTime = idle })), + schedulertest.WithConfig(schedulertest.ConfigWith(func(tw *scheduler.Tweakables) { tw.IdleTime = idle })), schedulertest.WithAfterStep(collectViolations(&violations)), ) diff --git a/chasm/lib/scheduler/schedulertest/library.go b/chasm/lib/scheduler/schedulertest/library.go index e8b827e321b..70c36954832 100644 --- a/chasm/lib/scheduler/schedulertest/library.go +++ b/chasm/lib/scheduler/schedulertest/library.go @@ -89,6 +89,15 @@ func DefaultConfig() *scheduler.Config { } } +// ConfigWith returns [DefaultConfig] with Tweakables adjusted by fn. +func ConfigWith(fn func(*scheduler.Tweakables)) *scheduler.Config { + cfg := DefaultConfig() + tweakables := scheduler.DefaultTweakables + fn(&tweakables) + cfg.Tweakables = func(string) scheduler.Tweakables { return tweakables } + return cfg +} + // NewRealSpecProcessor builds a real SpecProcessor with no-op metrics, matching // production spec processing so generated buffered starts and next-times are // realistic. diff --git a/cmd/tools/test/find_altered_tests.go b/cmd/tools/test/find_altered_tests.go index 600f5f9870b..0d5cee42104 100644 --- a/cmd/tools/test/find_altered_tests.go +++ b/cmd/tools/test/find_altered_tests.go @@ -33,10 +33,12 @@ import ( // FUNCTIONAL_TEST_XDC_ROOT // FUNCTIONAL_TEST_NDC_ROOT // INTEGRATION_TEST_DIRS +// PROPERTY_TEST_DIRS // UNIT_TEST_DIRS var CategoryDirs = map[string][]string{ "unit": {"./client", "./common", "./internal", "./service", "./temporal", "./tools", "./cmd"}, "integration": {"./common/persistence/tests", "./tools/tests", "./temporaltest"}, + "property": {"./chasm/lib/scheduler/schedulerproptest"}, "functional": {"./tests"}, "functional_ndc": {"./tests/ndc"}, "functional_xdc": {"./tests/xdc"}, diff --git a/docs/development/testing.md b/docs/development/testing.md index b087bffee52..81c13c4a61c 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -87,6 +87,125 @@ unless there is a reason not to. `make parallelize-tests` can be used to automatically add `t.Parallel()`. Use `//parallelize:ignore` to opt your test out of it. +## Property tests + +Property tests assert invariants against randomly-generated inputs instead of hand-written cases. +They use [rapid](https://pkg.go.dev/pgregory.net/rapid), which generates the inputs and, on +failure, shrinks them to a minimal counterexample. + +They run in their own tier, separate from unit tests, because they need a different runner +configuration; see [Why retries are disabled](#why-retries-are-disabled). + +The tier is every package directory whose name ends in `proptest`; see +[Adding a property test](#adding-a-property-test). + +### Running locally + +```bash +make property-test # rapid's default: 100 generated cases per property +make property-test RAPID_CHECKS=50000 # soak before pushing a change to a covered package +``` + +Or with plain `go test`. Note that `-rapid.*` are test *binary* flags, so the package has to come +before them — otherwise `go test` stops parsing and fails with `[setup failed]`: + +```bash +go test -tags test_dep ./chasm/lib/scheduler/schedulerproptest/ -run TestSchedulerProperties -rapid.checks=1000 +``` + +### Configuration + +`RAPID_CHECKS` sets the number of generated cases per property, and defaults to `100` — cheap +enough to run on every PR. Every rapid flag also has an environment variable form +(`-rapid.checks` / `RAPID_CHECKS`, and so on); see rapid's documentation for the full set. + +Seeds are **not** pinned in CI. Retries are disabled instead, which keeps a counterexample from +being discarded while still letting coverage accumulate across commits. + +### In CI + +| Job | Workflow | Gates merges | +| --- | --- | --- | +| `Property test` | `run-tests.yml`, every PR | intended | +| `Property test soak` | `property-test-soak.yml`, nightly 0600 UTC | no | + +Both run a single attempt. The soak also accepts a manual `workflow_dispatch` with a `rapid_checks` +input. + +`Property test` only blocks a merge once it is listed as a required status check in the repository's +branch protection settings; the workflow alone does not make it one. + +The soak is deliberately never a required check. A counterexample it finds is a real bug, but it is +found on an input that no particular PR introduced, so it must not block unrelated merges. + +### When a counterexample is found + +rapid prints the minimized counterexample along with a line naming both ways to reproduce it — the +*fail file* it just wrote, and the seed: + +``` +To reproduce, specify -run="" -rapid.failfile="" (or -rapid.seed=) +``` + +Either form works. The fail file path is relative to the package directory: + +```bash +go test -tags test_dep ./chasm/lib/scheduler/schedulerproptest/ -run TestSchedulerProperties \ + -rapid.failfile="testdata/rapid/TestSchedulerProperties/TestSchedulerProperties--.fail" +``` + +In CI the fail file is uploaded as a job artifact — `property-test-counterexamples`, or +`property-test-soak-counterexamples` for the nightly. Download it to reproduce locally. + +### Committing a counterexample + +rapid replays **every** fail file under `testdata/rapid//` before generating any random +cases, so a committed fail file becomes a permanent regression case enforced on every later run. +Fail file names are unique, so several can accumulate for the same test without colliding. + +**Commit the fail file in the same change as the fix.** It is a regression test for the bug it +encodes, and it belongs with the code that fixes it. + +**Do not commit a fail file for a bug that is not fixed yet.** The per-PR job replays it, so it +would fail on every PR and block all merges. Open an issue instead and attach the printed seed and +the fail file's contents. If the invariant has to be relaxed in the meantime, scope the exemption as +narrowly as possible and reference the issue from the code. + +The fail file goes next to the package under test, keyed by test name — exactly where rapid writes +it, so a file downloaded from a CI artifact is committed at the same path: + +``` +chasm/lib/scheduler/schedulerproptest/testdata/rapid/TestSchedulerProperties/TestSchedulerProperties--.fail +``` + +### Why retries are disabled + +`property-test-coverage` passes `--max-attempts=1` to the test runner, unlike the unit and +functional tiers, which retry up to `MAX_TEST_ATTEMPTS` times. **Do not re-enable retries for this +tier.** + +The test runner exits with the exit code of its *last* attempt, and each retry re-runs the test in a +fresh process, where rapid draws a new random seed. A retried property test therefore checks a +different set of inputs than the one that failed: the counterexample would be replaced by a +likely-passing run, and the job would go green having found a real bug. + +### Adding a property test + +**Put property tests in their own package directory, named with a `proptest` suffix.** The tier is +selected by directory name — `PROPERTY_TEST_DIRS` in the [Makefile](../../Makefile) matches any +directory ending in `proptest` — so creating one enrolls it automatically and excludes it from the +unit tier. Nothing needs registering. Also add it to the `property` category in +[find_altered_tests.go](../../cmd/tools/test/find_altered_tests.go), which is a separate list. + +The dedicated directory is not stylistic. The tier subtracts whole directories from +`UNIT_TEST_DIRS`, so a property test placed inside the package it exercises forces a choice between +pulling that package's entire unit test suite into the no-retry tier, or leaving the property test +in the retrying tier where its counterexamples are discarded. + +Because a sibling directory can only reach exported API, the package under test needs an exported +harness — see [schedulertest](../../chasm/lib/scheduler/schedulertest), which is a non-test package +precisely so property tests can import its driver. + ## Test helpers Test helpers can be found in the [common/testing](../../common/testing) package. From 5559f3e79cb2f5edf70d117582c4cdbdaa418eec Mon Sep 17 00:00:00 2001 From: chaptersix <13949480+chaptersix@users.noreply.github.com> Date: Fri, 13 Nov 2026 23:09:04 -0600 Subject: [PATCH 2/5] Remove stale known-bug suppressions from the scheduler property test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The deferred-start stranding that #13042 documented was already fixed when that PR merged. #13069 ("Keep CHASM schedule buffers progressing after discarded or failed starts") added the reenableDeferredStarts call in Invoker.addTasks, which returns deferred starts to Attempt == 0 when no pending or running start remains so the next ProcessBuffer pass reconsiders them. That commit is an ancestor of #13042, landing roughly seven hours and four commits earlier, so the skip and the invariant filter were stale on arrival — presumably written against an older base and rebased without re-checking. Both suppressions pass silently once the bug is gone, which is why they survived: nothing fails when the condition they describe stops happening. The filter was also broader than the bug it named, discarding every ledger-lost-start violation where nothing was running and all pending starts were deferred, not just the stranding case. - Un-skip TestLedger_DeferredStartsStrandedByPausedCompletion, and rewrite its comment, which described the broken behaviour as current, to explain the gap and name reenableDeferredStarts as the mechanism that closes it. - Delete checkInvariantsIgnoringKnownBugs, whose own comment asked for exactly this, and use schedulertest.CheckInvariantsHook so the property test now asserts the full invariant set. Verified with 300k generated cases (200k in one run plus five independent seeds at 20k each) and with the un-skipped test under -race -shuffle on. Co-Authored-By: Claude Opus 5 --- .../schedulerproptest/property_test.go | 23 +------------------ .../scheduler/schedulertest/ledger_test.go | 18 +++++++++------ 2 files changed, 12 insertions(+), 29 deletions(-) diff --git a/chasm/lib/scheduler/schedulerproptest/property_test.go b/chasm/lib/scheduler/schedulerproptest/property_test.go index fb0a252e332..3dfa3ec36b6 100644 --- a/chasm/lib/scheduler/schedulerproptest/property_test.go +++ b/chasm/lib/scheduler/schedulerproptest/property_test.go @@ -51,7 +51,7 @@ func TestSchedulerProperties(t *testing.T) { schedulertest.WithMaxSideEffectLag(idle/2), schedulertest.WithMaxRunDuration(idle/2), ) - hook := checkInvariantsIgnoringKnownBugs(rt) + hook := schedulertest.CheckInvariantsHook(rt) d.AfterStep = hook // Check the initial settled state too. @@ -141,24 +141,3 @@ func drawSchedule(rt *rapid.T, start time.Time, idle time.Duration) *schedulepb. return s } - -// checkInvariantsIgnoringKnownBugs is CheckInvariantsHook minus one known bug -// (TestLedger_DeferredStartsStrandedByPausedCompletion): a lost start where -// every pending start is deferred and nothing is running to re-enable them. -// Delete this once that bug is fixed. -func checkInvariantsIgnoringKnownBugs(rt *rapid.T) func(*schedulertest.Driver) { - var prev *schedulertest.Snapshot - return func(d *schedulertest.Driver) { - rt.Helper() - cur := d.Snapshot() - for _, v := range schedulertest.CheckInvariants(prev, cur) { - if v.Name == "ledger-lost-start" && cur.RunningWorkflows == 0 && - cur.DeferredStarts == len(cur.PendingStarts) { - continue - } - rt.Fatalf("scheduler invariant violation at step (now=%s): %s", - cur.Now.Format("2006-01-02T15:04:05"), v) - } - prev = &cur - } -} diff --git a/chasm/lib/scheduler/schedulertest/ledger_test.go b/chasm/lib/scheduler/schedulertest/ledger_test.go index 0df8a529124..a8e8f5f802f 100644 --- a/chasm/lib/scheduler/schedulertest/ledger_test.go +++ b/chasm/lib/scheduler/schedulertest/ledger_test.go @@ -160,14 +160,18 @@ func uniq(names []string) []string { return out } -// A start deferred behind a running workflow is only re-enabled by a workflow -// completion. If that completion arrives while the schedule is paused, -// ProcessBuffer drops the first start (paused) and re-defers the rest with -// nothing running, so no completion will ever re-enable them. Unpause doesn't -// re-process the buffer either; the starts (including a manual trigger, which -// pause allows) sit until the idle task closes the schedule. +// A start deferred behind a running workflow is normally re-enabled by that +// workflow completing. When the completion arrives while the schedule is +// paused, ProcessBuffer selects the first start, drops it because the schedule +// is paused, and re-defers the rest — leaving deferred starts with nothing +// running whose completion could ever wake them. +// +// Invoker.addTasks closes that gap: with no pending or running start it calls +// reenableDeferredStarts, which returns the deferred starts to Attempt == 0 so +// the next ProcessBuffer pass reconsiders them. This test guards that path, in +// particular that a manual trigger (which pause permits) still starts rather +// than sitting in the buffer until the idle task closes the schedule. func TestLedger_DeferredStartsStrandedByPausedCompletion(t *testing.T) { - t.Skip("known bug: deferred starts stranded after a completion while paused") const idle = time.Hour start := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) From 51bfd976425abcc5078836c94da4cc20e3550056 Mon Sep 17 00:00:00 2001 From: chaptersix <13949480+chaptersix@users.noreply.github.com> Date: Fri, 13 Nov 2026 23:13:58 -0600 Subject: [PATCH 3/5] Gate on the property test job, and stop treating fail files as regressions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Picked up from chaptersix/temporal#71, which built a property test tier independently and got two things right that this branch had wrong. The property-test job was not in the test-status aggregate gate's needs list, so it ran but could not block a merge: test-status reported success regardless of its result. Add it. More importantly, a rapid fail file is not a durable regression test, so testdata/rapid/ is now gitignored instead of committed. A fail file replays a raw sequence of draws, so adding, removing or reordering a draw in a generator makes the stored sequence decode to different values. Verified against rapid v1.3.0: with the original generator a stored fail file reproduces immediately ("failed after 0 tests"); after inserting one draw ahead of the existing one, the same file no longer reproduces and the test simply passes. rapid reports nothing — the protection disappears without anything failing to say so, which is the same silent-rot failure as the stale t.Skip removed in the previous commit. Replace the "commit the fail file" guidance in the docs, the Makefile and the soak workflow with pinning the case as a hand-written deterministic test, and add a "Deferring a fix" section that spells out why skips and invariant filters rot, using the #13042/#13069 sequence as the worked example. Also narrow the counterexample artifact glob to *.fail, matching #71. Deliberately not taken from #71, to keep this change small: the production scheduler fixes and their regressions, the chasmtest.TaskRunner harness extraction, the rapid state-machine rewrite of the property test, the migration work and its proto field, and the mutation-test tool. Its rapid-import discovery for PROPERTY_TEST_DIRS was also left out: it pairs with #71 leaving property packages in the unit tier, whereas this tier subtracts them, where auto-discovery would pull an entire production package's unit tests out of the retrying tier. Co-Authored-By: Claude Opus 5 --- .github/workflows/property-test-soak.yml | 10 ++-- .github/workflows/run-tests.yml | 3 +- .gitignore | 6 +++ Makefile | 8 ++-- docs/development/testing.md | 58 ++++++++++++++++-------- 5 files changed, 56 insertions(+), 29 deletions(-) diff --git a/.github/workflows/property-test-soak.yml b/.github/workflows/property-test-soak.yml index 47f696fd76b..d8826c86be6 100644 --- a/.github/workflows/property-test-soak.yml +++ b/.github/workflows/property-test-soak.yml @@ -7,10 +7,10 @@ name: Property Test Soak # # It is deliberately NOT a required check. A soak finding is a real bug, but it # is found on an input no PR introduced, so it should not block unrelated merges. -# When it fails, download the counterexample artifact and commit the .fail file -# under the package's testdata/rapid/ directory: rapid replays every such file -# ahead of the random cases, which turns the finding into a permanent regression -# case that the per-PR job then enforces. +# When it fails, download the counterexample artifact to reproduce the case +# locally, then pin it as a hand-written deterministic test. The fail files are +# gitignored rather than committed, because they replay a raw draw sequence and +# stop reproducing silently once a generator changes. on: schedule: @@ -61,7 +61,7 @@ jobs: uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: property-test-soak-counterexamples - path: "**/testdata/rapid/**" + path: "**/testdata/rapid/**/*.fail" if-no-files-found: ignore - name: Post-test reporting diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 0eaef0e9666..455366d3b82 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -360,7 +360,7 @@ jobs: uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: property-test-counterexamples - path: "**/testdata/rapid/**" + path: "**/testdata/rapid/**/*.fail" if-no-files-found: ignore - name: Post-test reporting @@ -542,6 +542,7 @@ jobs: needs: - misc-checks - unit-test + - property-test - integration-test - functional-test runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index f6e1955edc7..48b2dda4a29 100644 --- a/.gitignore +++ b/.gitignore @@ -48,3 +48,9 @@ # Ignoring AI agent files .agents/ + +# rapid failfiles replay raw draw sequences, so any change to a generator makes a +# stored file decode to different values and stop reproducing — silently, with no +# error. Pin regressions with hand-written tests instead. CI uploads failfiles as +# artifacts on failure for local reproduction. +**/testdata/rapid/ diff --git a/Makefile b/Makefile index 430e4dc974f..7ff2eb5c5b7 100644 --- a/Makefile +++ b/Makefile @@ -502,9 +502,11 @@ integration-test: clean-test-output # default and costs ~60ms; the nightly soak workflow raises it a long way. # # When a property fails, rapid writes a minimized reproducer to -# testdata/rapid//--.fail and replays every such -# file ahead of the random cases on subsequent runs. Commit those files to turn a -# counterexample into a permanent regression case. +# testdata/rapid//--.fail. Use it to reproduce the +# failure locally, then pin the case as a hand-written deterministic test. The +# fail files themselves are gitignored: they replay a raw draw sequence, so any +# change to a generator makes them decode to different values and stop +# reproducing the bug silently. # # See docs/development/testing.md ("Property tests") for how to reproduce a # counterexample, when to commit one, and how to add a package to this tier. diff --git a/docs/development/testing.md b/docs/development/testing.md index 81c13c4a61c..ab27ceb5bf6 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -157,26 +157,44 @@ go test -tags test_dep ./chasm/lib/scheduler/schedulerproptest/ -run TestSchedul In CI the fail file is uploaded as a job artifact — `property-test-counterexamples`, or `property-test-soak-counterexamples` for the nightly. Download it to reproduce locally. -### Committing a counterexample - -rapid replays **every** fail file under `testdata/rapid//` before generating any random -cases, so a committed fail file becomes a permanent regression case enforced on every later run. -Fail file names are unique, so several can accumulate for the same test without colliding. - -**Commit the fail file in the same change as the fix.** It is a regression test for the bug it -encodes, and it belongs with the code that fixes it. - -**Do not commit a fail file for a bug that is not fixed yet.** The per-PR job replays it, so it -would fail on every PR and block all merges. Open an issue instead and attach the printed seed and -the fail file's contents. If the invariant has to be relaxed in the meantime, scope the exemption as -narrowly as possible and reference the issue from the code. - -The fail file goes next to the package under test, keyed by test name — exactly where rapid writes -it, so a file downloaded from a CI artifact is committed at the same path: - -``` -chasm/lib/scheduler/schedulerproptest/testdata/rapid/TestSchedulerProperties/TestSchedulerProperties--.fail -``` +### Pinning a counterexample + +**Pin the counterexample as a hand-written deterministic test, in the same change as the fix.** +Translate the shrunk inputs into an explicit test — `TestLedger_DeferredStartsStrandedByPausedCompletion` +in [schedulertest](../../chasm/lib/scheduler/schedulertest) is an example — and put it next to the +other unit tests for the package, not in the property tier. + +**Do not commit the fail file.** `testdata/rapid/` is gitignored, because a fail file is not a +durable regression test: it replays a raw sequence of draws, so adding, removing or reordering a +draw in the generator makes the stored sequence decode to different values. It then stops +reproducing the bug **silently** — rapid reports no error, the replay simply passes, and the +protection is gone without anything failing to tell you. A hand-written test says what it means and +breaks loudly when behaviour changes. + +Use the fail file for what it is good at — reproducing the failure *now*, while you write the +real test. + +### Deferring a fix + +If the bug cannot be fixed immediately, the thing to avoid is any marker that passes silently once +the bug is gone. A `t.Skip` and a filter in the invariant hook both do, which is how this +repository ended up carrying a skip *and* an invariant exemption for a stranding bug that had +already been fixed on main seven hours earlier — the skip's own comment asked for its deletion, and +nothing deleted it because nothing broke when the bug disappeared. + +So, in order of preference: + +- Fix it. A shrunk counterexample is usually small, and the fix often is too. +- Open an issue, and paste in both the seed and the fail file's contents — a few lines of hex that + fully describe the case. Do not rely on the CI artifact; those expire. +- If the property has to keep running while the bug is open, constrain the **generator** rather + than filtering violations in the hook. A generator constraint is visible in the inputs and + self-documenting; a violation filter silently swallows every other bug of the same shape, not + just the one it names. +- If you must leave a marker, make it fail when the bug is fixed — for example a deterministic test + that asserts the current wrong outcome behind a `// BUG(#NNNNN):` comment, so whoever fixes the + bug is forced to update it. Put the issue number in any `t.Skip` message so it is at least + greppable. ### Why retries are disabled From 692460b2184ddd4538fa10e1463ea206f9783cd5 Mon Sep 17 00:00:00 2001 From: chaptersix <13949480+chaptersix@users.noreply.github.com> Date: Fri, 13 Nov 2026 23:23:36 -0600 Subject: [PATCH 4/5] Build the test engine on an interface so property failures shrink MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Raise the per-PR property tier to 1000 generated cases, and take the chasmtest.TB change from chaptersix/temporal#71 without the driver rewrite that PR builds on top of it. chasmtest.Engine took a *testing.T, so a driver built inside a rapid property had to be handed the outer *testing.T. Anything the harness itself reported — a gomock expectation, an engine assertion, an error-level test log — then failed the enclosing test instead of the generated case, unshrunk and unattributed. Only the explicit invariant checks, which were reported through the *rapid.T, could shrink. Declare chasmtest.TB as the subset of testing.TB the engine, gomock and testlogger actually need, and take it in NewEngine, Engine.t and InvariantCheck. *testing.T still satisfies it, so no caller changes; *rapid.T satisfies it too, including FailNow transitively through zaptest.TestingT, so testify's require keeps working. Widen the three scheduler invariant helpers and the handful of InvariantCheck literals that the narrower signature pinned to *testing.T. schedulertest.NewDriver, NewDriverFromMigration, newDriverScaffold and Driver.T take chasmtest.TB as well — the driver only ever used Helper and Fatalf — and the property test now builds its driver on rt. Verified by failing through the driver's own T inside a property: the case shrinks to the same minimal draw as an identical property with no driver (401 of 1..1000, over four consecutive runs). Harness nondeterminism can still occasionally leave a case unminimized, since start RPCs run on concurrent goroutines. Co-Authored-By: Claude Opus 5 --- .github/workflows/property-test-soak.yml | 2 +- Makefile | 4 +++- chasm/chasmtest/test_engine.go | 18 ++++++++++++++---- chasm/chasmtest/test_engine_test.go | 6 +++--- chasm/lib/scheduler/generator_wakeup_test.go | 2 +- chasm/lib/scheduler/helper_test.go | 2 +- chasm/lib/scheduler/invariant_test.go | 7 ++++--- .../schedulerproptest/property_test.go | 9 +++++---- chasm/lib/scheduler/schedulertest/driver.go | 9 ++++----- docs/development/testing.md | 4 ++-- 10 files changed, 38 insertions(+), 25 deletions(-) diff --git a/.github/workflows/property-test-soak.yml b/.github/workflows/property-test-soak.yml index d8826c86be6..a234d590d9b 100644 --- a/.github/workflows/property-test-soak.yml +++ b/.github/workflows/property-test-soak.yml @@ -1,6 +1,6 @@ name: Property Test Soak -# The per-PR Property test job runs rapid's default 100 checks so it stays cheap +# The per-PR Property test job runs 1000 checks so it stays cheap # enough to gate on. This workflow is where the input space actually gets # explored: the same tests, same single attempt, but orders of magnitude more # generated cases, on a fresh random seed each night. diff --git a/Makefile b/Makefile index 7ff2eb5c5b7..f85e88cb389 100644 --- a/Makefile +++ b/Makefile @@ -511,7 +511,9 @@ integration-test: clean-test-output # See docs/development/testing.md ("Property tests") for how to reproduce a # counterexample, when to commit one, and how to add a package to this tier. PROPERTY_TEST_MAX_ATTEMPTS := 1 -RAPID_CHECKS ?= 100 +# Generated cases per property. Above rapid's own default of 100, to catch more +# before a merge; the nightly soak workflow raises it much further. +RAPID_CHECKS ?= 1000 property-test: clean-test-output @printf $(COLOR) "Run property tests ($(RAPID_CHECKS) checks)..." diff --git a/chasm/chasmtest/test_engine.go b/chasm/chasmtest/test_engine.go index 8b33b40c3cf..aaa1d1d61b7 100644 --- a/chasm/chasmtest/test_engine.go +++ b/chasm/chasmtest/test_engine.go @@ -6,7 +6,6 @@ import ( "fmt" "maps" "sync" - "testing" "time" enumspb "go.temporal.io/api/enums/v1" @@ -25,19 +24,30 @@ import ( ) type ( + // TB is the subset of [testing.TB] the engine, gomock controllers and + // testlogger need. Both *testing.T and *rapid.T satisfy it, so an engine can + // be built inside a rapid property and have failures flow through rapid's + // shrinking. It is declared here, rather than imported from rapid, so this + // package does not depend on rapid. + TB interface { + testlogger.CleanupCapableT + Errorf(format string, args ...any) + Context() context.Context + } + EngineOption func(*Engine) // InvariantCheck validates an execution after a successful transition. The // node is clean, so checks may inspect its snapshot without affecting the // state being validated. - InvariantCheck func(*testing.T, *chasm.Node, chasm.RootComponent) + InvariantCheck func(TB, *chasm.Node, chasm.RootComponent) // Engine is a lightweight in memory CHASM engine for unit tests. It implements // [chasm.Engine] and supports the full set of conflict and reuse policies, as // well as blocking [PollComponent] with [NotifyExecution], matching the behavior // of the production engine as closely as possible without persistence or shard logic. Engine struct { - t *testing.T + t TB registry *chasm.Registry logger log.Logger metrics metrics.Handler @@ -107,7 +117,7 @@ var defaultTransitionOptions = chasm.TransitionOptions{ var _ chasm.Engine = (*Engine)(nil) func NewEngine( - t *testing.T, + t TB, registry *chasm.Registry, opts ...EngineOption, ) *Engine { diff --git a/chasm/chasmtest/test_engine_test.go b/chasm/chasmtest/test_engine_test.go index 48dc3c54708..390595d2ef7 100644 --- a/chasm/chasmtest/test_engine_test.go +++ b/chasm/chasmtest/test_engine_test.go @@ -50,7 +50,7 @@ func TestTasksArePhysicallyGenerated(t *testing.T) { func TestInvariantCheckRunsAfterSuccessfulTransitions(t *testing.T) { var calls []string check := func(name string) chasmtest.InvariantCheck { - return func(checkT *testing.T, node *chasm.Node, root chasm.RootComponent) { + return func(checkT chasmtest.TB, node *chasm.Node, root chasm.RootComponent) { checkT.Helper() require.Same(checkT, t, checkT) require.IsType(checkT, &tests.PayloadStore{}, root) @@ -122,7 +122,7 @@ func TestInvariantCheckRunsAfterSuccessfulTransitions(t *testing.T) { // sub-components its update adds must become nodes in the tree, not only exist in memory. func TestUpdateWithStartSyncsSubComponentsAddedByUpdate(t *testing.T) { var nodePaths []string - check := func(_ *testing.T, node *chasm.Node, _ chasm.RootComponent) { + check := func(_ chasmtest.TB, node *chasm.Node, _ chasm.RootComponent) { nodePaths = slices.Collect(maps.Keys(node.Snapshot(nil).Nodes)) } e, _ := startStoreWithEngineOptions(t, 0, []chasmtest.EngineOption{chasmtest.WithInvariantCheck(check)}) @@ -368,7 +368,7 @@ func TestRecordsCallerPrincipal(t *testing.T) { worker := &commonpb.Principal{Type: "jwt", Name: "worker"} var rootAttributes *persistencespb.ChasmComponentAttributes - captureRoot := func(_ *testing.T, node *chasm.Node, _ chasm.RootComponent) { + captureRoot := func(_ chasmtest.TB, node *chasm.Node, _ chasm.RootComponent) { rootAttributes = node.Snapshot(nil).Nodes[""].GetMetadata().GetComponentAttributes() } diff --git a/chasm/lib/scheduler/generator_wakeup_test.go b/chasm/lib/scheduler/generator_wakeup_test.go index c895cbb0a12..b35cbd75f03 100644 --- a/chasm/lib/scheduler/generator_wakeup_test.go +++ b/chasm/lib/scheduler/generator_wakeup_test.go @@ -54,7 +54,7 @@ func TestGeneratorNextWakeupTimeStaysAtPendingJitteredFire(t *testing.T) { require.NoError(t, err) var timerDeadlines []time.Time observeTimers := func(config *engineTestConfig) { - config.engineOpts = append(config.engineOpts, chasmtest.WithInvariantCheck(func(_ *testing.T, node *chasm.Node, _ chasm.RootComponent) { + config.engineOpts = append(config.engineOpts, chasmtest.WithInvariantCheck(func(_ chasmtest.TB, node *chasm.Node, _ chasm.RootComponent) { timerDeadlines = nil generator := node.Snapshot(nil).Nodes[generatorPath] for _, task := range generator.GetMetadata().GetComponentAttributes().GetPureTasks() { diff --git a/chasm/lib/scheduler/helper_test.go b/chasm/lib/scheduler/helper_test.go index b73ed03e28f..295a431d761 100644 --- a/chasm/lib/scheduler/helper_test.go +++ b/chasm/lib/scheduler/helper_test.go @@ -278,7 +278,7 @@ func newTestEngineContextFromConfig( require.NoError(t, registry.Register(newTestLibrary(logger, specProcessor))) config.engineOpts = append(config.engineOpts, chasmtest.WithInvariantCheck( - func(t *testing.T, node *chasm.Node, root chasm.RootComponent) { + func(t chasmtest.TB, node *chasm.Node, root chasm.RootComponent) { requireValidSchedulerState(t, registry, node, root) }, )) diff --git a/chasm/lib/scheduler/invariant_test.go b/chasm/lib/scheduler/invariant_test.go index b35b08be0f2..699eb70cc95 100644 --- a/chasm/lib/scheduler/invariant_test.go +++ b/chasm/lib/scheduler/invariant_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/require" "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/chasmtest" "go.temporal.io/server/chasm/lib/scheduler" "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" "google.golang.org/protobuf/types/known/timestamppb" @@ -139,7 +140,7 @@ func idleTaskCount(registry *chasm.Registry, node *chasm.Node) int { // Only asserted in this direction. Sentinels legitimately arm an idle task // without setting IdleCloseTime (see NewSentinel), so an armed task with no // declared deadline is not a defect. -func requireIdleCloseTimeBacked(t *testing.T, registry *chasm.Registry, node *chasm.Node, sched *scheduler.Scheduler) { +func requireIdleCloseTimeBacked(t chasmtest.TB, registry *chasm.Registry, node *chasm.Node, sched *scheduler.Scheduler) { t.Helper() if sched.GetSentinel() || sched.GetClosed() || sched.GetIdleCloseTime() == nil { @@ -153,7 +154,7 @@ func requireIdleCloseTimeBacked(t *testing.T, registry *chasm.Registry, node *ch } // requireNotStuck asserts the stuckness invariant against a clean tree. -func requireNotStuck(t *testing.T, node *chasm.Node, sched *scheduler.Scheduler) { +func requireNotStuck(t chasmtest.TB, node *chasm.Node, sched *scheduler.Scheduler) { t.Helper() total, byPath := liveTaskCount(node) @@ -162,7 +163,7 @@ func requireNotStuck(t *testing.T, node *chasm.Node, sched *scheduler.Scheduler) } func requireValidSchedulerState( - t *testing.T, + t chasmtest.TB, registry *chasm.Registry, node *chasm.Node, root chasm.RootComponent, diff --git a/chasm/lib/scheduler/schedulerproptest/property_test.go b/chasm/lib/scheduler/schedulerproptest/property_test.go index 3dfa3ec36b6..62913ff7069 100644 --- a/chasm/lib/scheduler/schedulerproptest/property_test.go +++ b/chasm/lib/scheduler/schedulerproptest/property_test.go @@ -28,9 +28,10 @@ import ( // failing operation sequence to a minimal reproducer — directly useful for the // "stuck open" bug class. // -// The outer *testing.T owns the engine scaffolding (it requires a real -// testing.TB); invariant violations are reported through the *rapid.T so rapid -// can shrink them. +// The engine scaffolding is built on the *rapid.T rather than the outer +// *testing.T, so a harness failure (a gomock expectation, an engine assertion, +// an error-level log) shrinks and is attributed to the generated case that +// caused it, exactly like an invariant violation. func TestSchedulerProperties(t *testing.T) { rapid.Check(t, func(rt *rapid.T) { start := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) @@ -39,7 +40,7 @@ func TestSchedulerProperties(t *testing.T) { idle := rapid.SampledFrom([]time.Duration{10 * time.Minute, time.Hour, 2 * time.Hour}).Draw(rt, "idleTime") sched := drawSchedule(rt, start, idle) - d := schedulertest.NewDriver(t, + d := schedulertest.NewDriver(rt, schedulertest.WithStartTime(start), schedulertest.WithSchedule(sched), schedulertest.WithConfig(schedulertest.ConfigWith(func(tw *scheduler.Tweakables) { diff --git a/chasm/lib/scheduler/schedulertest/driver.go b/chasm/lib/scheduler/schedulertest/driver.go index 93e8b94cb89..a2c04a50e27 100644 --- a/chasm/lib/scheduler/schedulertest/driver.go +++ b/chasm/lib/scheduler/schedulertest/driver.go @@ -6,7 +6,6 @@ import ( "fmt" "maps" "sync" - "testing" "time" commonpb "go.temporal.io/api/common/v1" @@ -52,7 +51,7 @@ var maxKeyFireTime = tasks.MaximumKey.FireTime // re-arms a generator task), so callers bound the run with maxSteps; quiescence is // reached only for paused/exhausted/closed schedules. type Driver struct { - T *testing.T + T chasmtest.TB Engine *chasmtest.Engine Registry *chasm.Registry TimeSource *clock.EventTimeSource @@ -153,7 +152,7 @@ func WithMaxRunDuration(d time.Duration) DriverOption { // NewDriver constructs a Driver and starts a scheduler execution via the // production CreateScheduler path. startTime defaults to a fixed instant for // determinism. -func NewDriver(t *testing.T, opts ...DriverOption) *Driver { +func NewDriver(t chasmtest.TB, opts ...DriverOption) *Driver { t.Helper() d, cfg := newDriverScaffold(t, opts...) @@ -175,7 +174,7 @@ func NewDriver(t *testing.T, opts ...DriverOption) *Driver { // NewDriverFromMigration constructs a Driver and starts a scheduler execution // from a migrated V1 state via the production CreateSchedulerFromMigration path. // This is the V1->V2 import side of a migration round trip. -func NewDriverFromMigration(t *testing.T, req *schedulerpb.CreateFromMigrationStateRequest, opts ...DriverOption) *Driver { +func NewDriverFromMigration(t chasmtest.TB, req *schedulerpb.CreateFromMigrationStateRequest, opts ...DriverOption) *Driver { t.Helper() d, _ := newDriverScaffold(t, opts...) d.startExecution(func(mctx chasm.MutableContext) (*scheduler.Scheduler, error) { @@ -186,7 +185,7 @@ func NewDriverFromMigration(t *testing.T, req *schedulerpb.CreateFromMigrationSt // newDriverScaffold builds the engine, registry, clients, and time source, but // does not start an execution. -func newDriverScaffold(t *testing.T, opts ...DriverOption) (*Driver, *driverConfig) { +func newDriverScaffold(t chasmtest.TB, opts ...DriverOption) (*Driver, *driverConfig) { t.Helper() cfg := &driverConfig{ diff --git a/docs/development/testing.md b/docs/development/testing.md index ab27ceb5bf6..1163d3557e4 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -102,7 +102,7 @@ The tier is every package directory whose name ends in `proptest`; see ### Running locally ```bash -make property-test # rapid's default: 100 generated cases per property +make property-test # 1000 generated cases per property make property-test RAPID_CHECKS=50000 # soak before pushing a change to a covered package ``` @@ -115,7 +115,7 @@ go test -tags test_dep ./chasm/lib/scheduler/schedulerproptest/ -run TestSchedul ### Configuration -`RAPID_CHECKS` sets the number of generated cases per property, and defaults to `100` — cheap +`RAPID_CHECKS` sets the number of generated cases per property, and defaults to `1000` — cheap enough to run on every PR. Every rapid flag also has an environment variable form (`-rapid.checks` / `RAPID_CHECKS`, and so on); see rapid's documentation for the full set. From c4e43e6efdf9d430c184b4cd03849f286bbd57f4 Mon Sep 17 00:00:00 2001 From: chaptersix <13949480+chaptersix@users.noreply.github.com> Date: Fri, 13 Nov 2026 23:37:02 -0600 Subject: [PATCH 5/5] Trigger CI Empty commit to start a workflow run now that Actions are re-enabled on the repository. Co-Authored-By: Claude Opus 5