Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 74 additions & 0 deletions .github/workflows/property-test-soak.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
name: Property Test Soak

# The per-PR Property test job runs 1000 checks so it stays cheap
# enough to gate on. This workflow is where the input space actually gets
# explored: the same tests, same single attempt, but orders of magnitude more
# generated cases, on a fresh random seed each night.
#
# It is deliberately NOT a required check. A soak finding is a real bug, but it
# is found on an input no PR introduced, so it should not block unrelated merges.
# When it fails, download the counterexample artifact to reproduce the case
# locally, then pin it as a hand-written deterministic test. The fail files are
# gitignored rather than committed, because they replay a raw draw sequence and
# stop reproducing silently once a generator changes.

on:
schedule:
# Runs at 0600 UTC daily (night in the US), before the other daily jobs.
- cron: "0 6 * * *"
workflow_dispatch: # Allow manual trigger
inputs:
rapid_checks:
description: Number of generated cases per property
required: false
default: "100000"

permissions:
contents: read

concurrency:
group: property-test-soak-${{ github.ref }}
cancel-in-progress: true

jobs:
soak:
name: Property test soak
runs-on: ubuntu-24.04-arm
env:
RAPID_CHECKS: ${{ inputs.rapid_checks || '100000' }}
TEST_TIMEOUT: 50m
GITHUB_TIMEOUT: 60
# Single attempt, as in the per-PR job: a retry would re-draw the seed and
# discard the counterexample the soak exists to find.
MAX_TEST_ATTEMPTS: 1
steps:
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
token: ${{ secrets.GITHUB_TOKEN }}

- name: Prepare Go
uses: ./.github/actions/prepare-go

- name: Run property test soak
timeout-minutes: ${{ fromJSON(env.GITHUB_TIMEOUT) }}
run: |
echo "::notice::Soaking with RAPID_CHECKS=$RAPID_CHECKS"
export TEST_RUNNER_TIMEOUT="$(("$GITHUB_TIMEOUT" - 1))m"
./develop/github/monitor_test.sh make property-test-coverage

- name: Upload property test counterexamples
if: failure()
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
with:
name: property-test-soak-counterexamples
path: "**/testdata/rapid/**/*.fail"
if-no-files-found: ignore

- name: Post-test reporting
if: always()
uses: ./.github/actions/post-test-reporting
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
job_name: Property test soak
artifact_name_suffix: property-test-soak
49 changes: 49 additions & 0 deletions .github/workflows/run-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -324,6 +324,54 @@ jobs:
path: ${{ env.LEAK_OUTPUT_DIR }}
if-no-files-found: ignore

property-test:
name: Property test
needs: [pre-build]
runs-on: ubuntu-24.04-arm
env:
TEST_TIMEOUT: 5m
GITHUB_TIMEOUT: 10
# Property tests run a single attempt: a retry would re-draw rapid's random
# seed and replace a real counterexample with a different, likely passing,
# input set. The Makefile locks the attempt count, and this keeps
# post-test-reporting's "fewer junit files than attempts means crash"
# heuristic correct for a one-attempt job.
MAX_TEST_ATTEMPTS: 1
steps:
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
token: ${{ secrets.GITHUB_TOKEN }}
ref: ${{ env.COMMIT }}

- name: Prepare Go
uses: ./.github/actions/prepare-go

- name: Run property tests
timeout-minutes: ${{ fromJSON(env.GITHUB_TIMEOUT) }}
run: |
export TEST_RUNNER_TIMEOUT="$(("$GITHUB_TIMEOUT" - 1))m"
./develop/github/monitor_test.sh make property-test-coverage

# rapid writes a minimized reproducer next to the failing package. Without
# this it is discarded with the runner and the counterexample is lost.
# Commit a downloaded .fail file to replay it on every later run.
- name: Upload property test counterexamples
if: failure()
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
with:
name: property-test-counterexamples
path: "**/testdata/rapid/**/*.fail"
if-no-files-found: ignore

- name: Post-test reporting
if: always()
uses: ./.github/actions/post-test-reporting
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
job_name: Property test
artifact_name_suffix: property-test

integration-test:
name: Integration test
needs: [pre-build]
Expand Down Expand Up @@ -494,6 +542,7 @@ jobs:
needs:
- misc-checks
- unit-test
- property-test
- integration-test
- functional-test
runs-on: ubuntu-latest
Expand Down
6 changes: 6 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -48,3 +48,9 @@

# Ignoring AI agent files
.agents/

# rapid failfiles replay raw draw sequences, so any change to a generator makes a
# stored file decode to different values and stop reproducing — silently, with no
# error. Pin regressions with hand-written tests instead. CI uploads failfiles as
# artifacts on failure for local reproduction.
**/testdata/rapid/
50 changes: 48 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -127,9 +127,17 @@ MIXED_BRAIN_TEST_ROOT := ./tests/mixedbrain
DB_INTEGRATION_TEST_ROOT := ./common/persistence/tests
DB_TOOL_INTEGRATION_TEST_ROOT := ./tools/tests
INTEGRATION_TEST_DIRS := $(DB_INTEGRATION_TEST_ROOT) $(DB_TOOL_INTEGRATION_TEST_ROOT) ./temporaltest
# Property tests run in their own tier so they can be run without retries; see
# the Property tests section below for why. The tier is selected by directory
# name: any package directory ending in "proptest" joins it, and is filtered out
# of UNIT_TEST_DIRS below so it does not also run in the retrying unit tier,
# where a counterexample would be masked by a retry. Deriving the list from a
# naming convention means a new property test package cannot be silently left in
# the unit tier by forgetting to register it here.
PROPERTY_TEST_DIRS := $(patsubst %/,%,$(filter %proptest/,$(TEST_DIRS)))
TESTCORE_UNITTESTS := ./tests/testcore
ifeq ($(UNIT_TEST_DIRS),)
UNIT_TEST_DIRS := $(filter-out $(FUNCTIONAL_TEST_ROOT)% $(FUNCTIONAL_TEST_XDC_ROOT)% $(FUNCTIONAL_TEST_NDC_ROOT)% $(MIXED_BRAIN_TEST_ROOT)% $(DB_INTEGRATION_TEST_ROOT)% $(DB_TOOL_INTEGRATION_TEST_ROOT)% ./temporaltest%,$(TEST_DIRS))
UNIT_TEST_DIRS := $(filter-out $(FUNCTIONAL_TEST_ROOT)% $(FUNCTIONAL_TEST_XDC_ROOT)% $(FUNCTIONAL_TEST_NDC_ROOT)% $(MIXED_BRAIN_TEST_ROOT)% $(DB_INTEGRATION_TEST_ROOT)% $(DB_TOOL_INTEGRATION_TEST_ROOT)% ./temporaltest% $(addsuffix %,$(PROPERTY_TEST_DIRS)),$(TEST_DIRS))

# Testcore unit tests are filtered out by the FUNCTIONAL_TEST_ROOT pattern, need to add them back manually.
UNIT_TEST_DIRS += $(TESTCORE_UNITTESTS)
Expand Down Expand Up @@ -481,6 +489,37 @@ integration-test: clean-test-output
@CGO_ENABLED=$(TEST_CGO_ENABLED) go test $(INTEGRATION_TEST_DIRS) $(COMPILED_TEST_ARGS) 2>&1 | tee -a test.log
@$(MAKE) verify-test-log

# Property tests check invariants against randomly-generated inputs, so they need
# a different runner configuration than ordinary unit tests:
#
# 1. They must NOT be retried. test-runner exits with the exit code of its last
# attempt only, and a retry re-runs the test in a fresh process where rapid
# draws a new random seed. A real counterexample would therefore be replaced
# by a different (likely passing) input set and the job would go green. The
# attempt count is locked with := so that the MAX_TEST_ATTEMPTS environment
# variable set in the CI workflow cannot re-enable retries here.
# 2. The number of generated cases is a knob. The default matches rapid's own
# default and costs ~60ms; the nightly soak workflow raises it a long way.
#
# When a property fails, rapid writes a minimized reproducer to
# testdata/rapid/<TestName>/<TestName>-<ts>-<pid>.fail. Use it to reproduce the
# failure locally, then pin the case as a hand-written deterministic test. The
# fail files themselves are gitignored: they replay a raw draw sequence, so any
# change to a generator makes them decode to different values and stop
# reproducing the bug silently.
#
# See docs/development/testing.md ("Property tests") for how to reproduce a
# counterexample, when to commit one, and how to add a package to this tier.
PROPERTY_TEST_MAX_ATTEMPTS := 1
# Generated cases per property. Above rapid's own default of 100, to catch more
# before a merge; the nightly soak workflow raises it much further.
RAPID_CHECKS ?= 1000

property-test: clean-test-output
@printf $(COLOR) "Run property tests ($(RAPID_CHECKS) checks)..."
@RAPID_CHECKS=$(RAPID_CHECKS) CGO_ENABLED=$(TEST_CGO_ENABLED) go test $(PROPERTY_TEST_DIRS) $(COMPILED_TEST_ARGS) 2>&1 | tee -a test.log
@$(MAKE) verify-test-log

functional-test: clean-test-output
@printf $(COLOR) "Run functional tests..."
@CGO_ENABLED=$(TEST_CGO_ENABLED) go test $(FUNCTIONAL_TEST_ROOT) $(COMPILED_TEST_ARGS) -persistenceType=$(PERSISTENCE_TYPE) -persistenceDriver=$(PERSISTENCE_DRIVER) 2>&1 | tee -a test.log
Expand Down Expand Up @@ -521,7 +560,7 @@ verify-test-log:
@grep -q "^ok" test.log || (echo "TEST FAILURE: no passing test found in test.log" && exit 1)
@! grep -q "^--- FAIL" test.log || (echo "TEST FAILURE: failing test found in test.log" && exit 1)

test: unit-test integration-test functional-test
test: unit-test property-test integration-test functional-test

##### Coverage & Reporting #####
$(TEST_OUTPUT_ROOT):
Expand All @@ -544,6 +583,13 @@ integration-test-coverage: prepare-coverage-test
CGO_ENABLED=$(TEST_CGO_ENABLED) go run ./cmd/tools/test-runner test --gotestsum-path=$(GOTESTSUM) --max-attempts=$(MAX_TEST_ATTEMPTS) $(TEST_RUNNER_TIMEOUT_ARG) --junitfile=$(NEW_REPORT) -- \
$(COMPILED_TEST_ARGS) -coverprofile=$(NEW_COVER_PROFILE) $(INTEGRATION_TEST_DIRS)

# Runs through test-runner (not plain go test) to keep the JUnit report, coverage
# and flaky-test reporting, but with retries disabled. See PROPERTY_TEST_DIRS.
property-test-coverage: prepare-coverage-test
@printf $(COLOR) "Run property tests with coverage ($(RAPID_CHECKS) checks)..."
RAPID_CHECKS=$(RAPID_CHECKS) CGO_ENABLED=$(TEST_CGO_ENABLED) go run ./cmd/tools/test-runner test --gotestsum-path=$(GOTESTSUM) --max-attempts=$(PROPERTY_TEST_MAX_ATTEMPTS) $(TEST_RUNNER_TIMEOUT_ARG) --junitfile=$(NEW_REPORT) -- \
$(COMPILED_TEST_ARGS) -coverprofile=$(NEW_COVER_PROFILE) $(PROPERTY_TEST_DIRS)

# MUST use the same build flags as functional-test-coverage and functional-test-{xdc,ndc}-coverage for best build caching.
pre-build-functional-test-coverage: prepare-coverage-test
CGO_ENABLED=$(TEST_CGO_ENABLED) go test -c -cover -o /dev/null $(COMPILED_TEST_ARGS) $(COVERPKG_FLAG) $(FUNCTIONAL_TEST_ROOT)
Expand Down
18 changes: 14 additions & 4 deletions chasm/chasmtest/test_engine.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import (
"fmt"
"maps"
"sync"
"testing"
"time"

enumspb "go.temporal.io/api/enums/v1"
Expand All @@ -25,19 +24,30 @@ import (
)

type (
// TB is the subset of [testing.TB] the engine, gomock controllers and
// testlogger need. Both *testing.T and *rapid.T satisfy it, so an engine can
// be built inside a rapid property and have failures flow through rapid's
// shrinking. It is declared here, rather than imported from rapid, so this
// package does not depend on rapid.
TB interface {
testlogger.CleanupCapableT
Errorf(format string, args ...any)
Context() context.Context
}

EngineOption func(*Engine)

// InvariantCheck validates an execution after a successful transition. The
// node is clean, so checks may inspect its snapshot without affecting the
// state being validated.
InvariantCheck func(*testing.T, *chasm.Node, chasm.RootComponent)
InvariantCheck func(TB, *chasm.Node, chasm.RootComponent)

// Engine is a lightweight in memory CHASM engine for unit tests. It implements
// [chasm.Engine] and supports the full set of conflict and reuse policies, as
// well as blocking [PollComponent] with [NotifyExecution], matching the behavior
// of the production engine as closely as possible without persistence or shard logic.
Engine struct {
t *testing.T
t TB
registry *chasm.Registry
logger log.Logger
metrics metrics.Handler
Expand Down Expand Up @@ -107,7 +117,7 @@ var defaultTransitionOptions = chasm.TransitionOptions{
var _ chasm.Engine = (*Engine)(nil)

func NewEngine(
t *testing.T,
t TB,
registry *chasm.Registry,
opts ...EngineOption,
) *Engine {
Expand Down
6 changes: 3 additions & 3 deletions chasm/chasmtest/test_engine_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ func TestTasksArePhysicallyGenerated(t *testing.T) {
func TestInvariantCheckRunsAfterSuccessfulTransitions(t *testing.T) {
var calls []string
check := func(name string) chasmtest.InvariantCheck {
return func(checkT *testing.T, node *chasm.Node, root chasm.RootComponent) {
return func(checkT chasmtest.TB, node *chasm.Node, root chasm.RootComponent) {
checkT.Helper()
require.Same(checkT, t, checkT)
require.IsType(checkT, &tests.PayloadStore{}, root)
Expand Down Expand Up @@ -122,7 +122,7 @@ func TestInvariantCheckRunsAfterSuccessfulTransitions(t *testing.T) {
// sub-components its update adds must become nodes in the tree, not only exist in memory.
func TestUpdateWithStartSyncsSubComponentsAddedByUpdate(t *testing.T) {
var nodePaths []string
check := func(_ *testing.T, node *chasm.Node, _ chasm.RootComponent) {
check := func(_ chasmtest.TB, node *chasm.Node, _ chasm.RootComponent) {
nodePaths = slices.Collect(maps.Keys(node.Snapshot(nil).Nodes))
}
e, _ := startStoreWithEngineOptions(t, 0, []chasmtest.EngineOption{chasmtest.WithInvariantCheck(check)})
Expand Down Expand Up @@ -368,7 +368,7 @@ func TestRecordsCallerPrincipal(t *testing.T) {
worker := &commonpb.Principal{Type: "jwt", Name: "worker"}

var rootAttributes *persistencespb.ChasmComponentAttributes
captureRoot := func(_ *testing.T, node *chasm.Node, _ chasm.RootComponent) {
captureRoot := func(_ chasmtest.TB, node *chasm.Node, _ chasm.RootComponent) {
rootAttributes = node.Snapshot(nil).Nodes[""].GetMetadata().GetComponentAttributes()
}

Expand Down
2 changes: 1 addition & 1 deletion chasm/lib/scheduler/generator_wakeup_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ func TestGeneratorNextWakeupTimeStaysAtPendingJitteredFire(t *testing.T) {
require.NoError(t, err)
var timerDeadlines []time.Time
observeTimers := func(config *engineTestConfig) {
config.engineOpts = append(config.engineOpts, chasmtest.WithInvariantCheck(func(_ *testing.T, node *chasm.Node, _ chasm.RootComponent) {
config.engineOpts = append(config.engineOpts, chasmtest.WithInvariantCheck(func(_ chasmtest.TB, node *chasm.Node, _ chasm.RootComponent) {
timerDeadlines = nil
generator := node.Snapshot(nil).Nodes[generatorPath]
for _, task := range generator.GetMetadata().GetComponentAttributes().GetPureTasks() {
Expand Down
2 changes: 1 addition & 1 deletion chasm/lib/scheduler/helper_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -278,7 +278,7 @@ func newTestEngineContextFromConfig(
require.NoError(t, registry.Register(newTestLibrary(logger, specProcessor)))

config.engineOpts = append(config.engineOpts, chasmtest.WithInvariantCheck(
func(t *testing.T, node *chasm.Node, root chasm.RootComponent) {
func(t chasmtest.TB, node *chasm.Node, root chasm.RootComponent) {
requireValidSchedulerState(t, registry, node, root)
},
))
Expand Down
7 changes: 4 additions & 3 deletions chasm/lib/scheduler/invariant_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (

"github.com/stretchr/testify/require"
"go.temporal.io/server/chasm"
"go.temporal.io/server/chasm/chasmtest"
"go.temporal.io/server/chasm/lib/scheduler"
"go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1"
"google.golang.org/protobuf/types/known/timestamppb"
Expand Down Expand Up @@ -139,7 +140,7 @@ func idleTaskCount(registry *chasm.Registry, node *chasm.Node) int {
// Only asserted in this direction. Sentinels legitimately arm an idle task
// without setting IdleCloseTime (see NewSentinel), so an armed task with no
// declared deadline is not a defect.
func requireIdleCloseTimeBacked(t *testing.T, registry *chasm.Registry, node *chasm.Node, sched *scheduler.Scheduler) {
func requireIdleCloseTimeBacked(t chasmtest.TB, registry *chasm.Registry, node *chasm.Node, sched *scheduler.Scheduler) {
t.Helper()

if sched.GetSentinel() || sched.GetClosed() || sched.GetIdleCloseTime() == nil {
Expand All @@ -153,7 +154,7 @@ func requireIdleCloseTimeBacked(t *testing.T, registry *chasm.Registry, node *ch
}

// requireNotStuck asserts the stuckness invariant against a clean tree.
func requireNotStuck(t *testing.T, node *chasm.Node, sched *scheduler.Scheduler) {
func requireNotStuck(t chasmtest.TB, node *chasm.Node, sched *scheduler.Scheduler) {
t.Helper()

total, byPath := liveTaskCount(node)
Expand All @@ -162,7 +163,7 @@ func requireNotStuck(t *testing.T, node *chasm.Node, sched *scheduler.Scheduler)
}

func requireValidSchedulerState(
t *testing.T,
t chasmtest.TB,
registry *chasm.Registry,
node *chasm.Node,
root chasm.RootComponent,
Expand Down
Loading
Loading