diff --git a/loadgen/kitchen_sink_executor.go b/loadgen/kitchen_sink_executor.go index 3bd4ce88..9146b8f6 100644 --- a/loadgen/kitchen_sink_executor.go +++ b/loadgen/kitchen_sink_executor.go @@ -2,6 +2,7 @@ package loadgen import ( "context" + "errors" "github.com/golang/protobuf/proto" "github.com/temporalio/omes/loadgen/kitchensink" @@ -10,6 +11,9 @@ import ( type KitchenSinkExecutor struct { TestInput *kitchensink.TestInput + // RecoverOriginalStart retains the iteration request and acknowledged execution across retries. + RecoverOriginalStart bool + // Called once on start PrepareTestInput func(context.Context, ScenarioInfo, *kitchensink.TestInput) error @@ -27,20 +31,54 @@ func (k KitchenSinkExecutor) Run(ctx context.Context, info ScenarioInfo) error { // Create generic executor and run it ge := &GenericExecutor{ Execute: func(ctx context.Context, run *Run) error { - options := run.DefaultKitchenSinkWorkflowOptions() - testInputClone, ok := proto.Clone(k.TestInput).(*kitchensink.TestInput) - if !ok { - panic("failed to clone test input") + if !k.RecoverOriginalStart { + options, err := k.iterationOptions(ctx, run) + if err != nil { + return err + } + return run.ExecuteKitchenSinkWorkflow(ctx, &options) } - options.Params = testInputClone - if k.UpdateWorkflowOptions != nil { - err := k.UpdateWorkflowOptions(ctx, run, &options) + // Retries reuse the first attempt's request instead of constructing a new one. + if run.kitchenSinkExecution == nil { + options, err := k.iterationOptions(ctx, run) if err != nil { return err } + if seq := options.Params.GetClientSequence(); len(seq.GetActionSets()) > 0 { + return errors.New("original start recovery does not support client sequences") + } + run.kitchenSinkExecution = freezeKitchenSinkExecution(run, options) } - return run.ExecuteKitchenSinkWorkflow(ctx, &options) + frozen := run.kitchenSinkExecution + return run.executeKitchenSinkWorkflow(ctx, &frozen.options, frozen.executor) }, } return ge.Run(ctx, info) } + +func (k KitchenSinkExecutor) iterationOptions(ctx context.Context, run *Run) (KitchenSinkWorkflowOptions, error) { + options := run.DefaultKitchenSinkWorkflowOptions() + testInputClone, ok := proto.Clone(k.TestInput).(*kitchensink.TestInput) + if !ok { + panic("failed to clone test input") + } + options.Params = testInputClone + if k.UpdateWorkflowOptions != nil { + if err := k.UpdateWorkflowOptions(ctx, run, &options); err != nil { + return options, err + } + } + return options, nil +} + +type kitchenSinkExecution struct { + options KitchenSinkWorkflowOptions + executor *kitchensink.ClientActionsExecutor +} + +func freezeKitchenSinkExecution(run *Run, options KitchenSinkWorkflowOptions) *kitchenSinkExecution { + options.Params = proto.Clone(options.Params).(*kitchensink.TestInput) + executor := run.newClientActionsExecutor(&options) + executor.RecoverOriginalStart = true + return &kitchenSinkExecution{options: options, executor: executor} +} diff --git a/loadgen/kitchen_sink_original_start_test.go b/loadgen/kitchen_sink_original_start_test.go new file mode 100644 index 00000000..7d70f228 --- /dev/null +++ b/loadgen/kitchen_sink_original_start_test.go @@ -0,0 +1,109 @@ +package loadgen + +import ( + "context" + "fmt" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + "github.com/temporalio/omes/loadgen/kitchensink" + "go.temporal.io/api/serviceerror" + "go.temporal.io/sdk/client" + "go.uber.org/zap" +) + +type startInputsClient struct { + client.Client + ids []string + inputs []int32 +} + +func (c *startInputsClient) ExecuteWorkflow(ctx context.Context, opts client.StartWorkflowOptions, _ interface{}, args ...interface{}) (client.WorkflowRun, error) { + c.ids = append(c.ids, opts.ID) + c.inputs = append(c.inputs, args[0].(*kitchensink.WorkflowInput).ExpectedSignalCount) + if len(c.ids) == 1 { + return nil, serviceerror.NewDeadlineExceeded("start response unavailable") + } + return completedOriginalRun{id: opts.ID}, nil +} + +type completedOriginalRun struct { + client.WorkflowRun + id string +} + +func (r completedOriginalRun) GetID() string { return r.id } +func (r completedOriginalRun) GetRunID() string { return "original-run" } +func (r completedOriginalRun) Get(context.Context, interface{}) error { return nil } + +func TestKitchenSinkOriginalStartFreezesRetryConstruction(t *testing.T) { + cl := &startInputsClient{} + generation := int32(0) + executor := KitchenSinkExecutor{ + TestInput: &kitchensink.TestInput{ + WorkflowInput: &kitchensink.WorkflowInput{}, + }, + UpdateWorkflowOptions: func(ctx context.Context, r *Run, o *KitchenSinkWorkflowOptions) error { + generation++ + o.StartOptions.ID = uuid.NewString() + o.Params.WorkflowInput.ExpectedSignalCount = generation + return nil + }, + } + executor.RecoverOriginalStart = true + + info := ScenarioInfo{ + Client: cl, + Namespace: "default", + RunID: "frozen", + ExecutionID: "same", + Logger: zap.NewNop().Sugar(), + MetricsHandler: client.MetricsNopHandler, + Configuration: RunConfiguration{ + Iterations: 1, + MaxConcurrent: 1, + MaxIterationAttempts: 2, + DoNotRegisterSearchAttributes: true, + }, + } + require.NoError(t, executor.Run(t.Context(), info)) + require.Len(t, cl.ids, 2) + require.Equal(t, cl.ids[0], cl.ids[1]) + require.Equal(t, []int32{1, 1}, cl.inputs) +} + +func TestKitchenSinkOriginalStartRejectsClientSequence(t *testing.T) { + for _, configuredByCallback := range []bool{false, true} { + t.Run(fmt.Sprintf("configured_by_callback=%t", configuredByCallback), func(t *testing.T) { + cl := &startInputsClient{} + seq := &kitchensink.ClientSequence{ActionSets: []*kitchensink.ClientActionSet{{}}} + executor := KitchenSinkExecutor{ + RecoverOriginalStart: true, + TestInput: &kitchensink.TestInput{WorkflowInput: &kitchensink.WorkflowInput{}}, + } + if configuredByCallback { + executor.UpdateWorkflowOptions = func(_ context.Context, _ *Run, o *KitchenSinkWorkflowOptions) error { + o.Params.ClientSequence = seq + return nil + } + } else { + executor.TestInput.ClientSequence = seq + } + info := ScenarioInfo{ + Client: cl, + Namespace: "default", + Logger: zap.NewNop().Sugar(), + MetricsHandler: client.MetricsNopHandler, + Configuration: RunConfiguration{ + Iterations: 1, + MaxConcurrent: 1, + DoNotRegisterSearchAttributes: true, + }, + } + err := executor.Run(t.Context(), info) + require.ErrorContains(t, err, "original start recovery does not support client sequences") + require.Empty(t, cl.ids, "invalid recovery must be rejected before starting a workflow") + }) + } +} diff --git a/loadgen/kitchensink/client_action_executor.go b/loadgen/kitchensink/client_action_executor.go index 29f9c245..f554aacf 100644 --- a/loadgen/kitchensink/client_action_executor.go +++ b/loadgen/kitchensink/client_action_executor.go @@ -8,6 +8,7 @@ import ( "github.com/google/uuid" enumspb "go.temporal.io/api/enums/v1" + "go.temporal.io/api/serviceerror" "go.temporal.io/sdk/client" "go.temporal.io/sdk/converter" "go.temporal.io/sdk/workflow" @@ -22,6 +23,9 @@ type ClientActionsExecutor struct { WorkflowInput *WorkflowInput Handle client.WorkflowRun runID string + // RecoverOriginalStart rejects duplicate starts and retains acknowledged run/update handles. + RecoverOriginalStart bool + originalStart *originalStart } func (e *ClientActionsExecutor) Start( @@ -29,6 +33,27 @@ func (e *ClientActionsExecutor) Start( withStartAction *WithStartClientAction, ) error { var err error + if e.RecoverOriginalStart { + if err := ctx.Err(); err != nil { + return err + } + if e.originalStart == nil { + if withStartAction.GetDoSignal() != nil { + return errors.New("original start recovery does not support Signal-with-Start") + } + e.freezeStart(withStartAction) + } + withStartAction = e.originalStart.action + if e.originalStart.update != nil { + if err := e.originalStart.update.Get(ctx, nil); err != nil { + return err + } + } + if e.Handle != nil { + e.runID = e.Handle.GetRunID() + return nil + } + } if withStartAction == nil { e.Handle, err = e.Client.ExecuteWorkflow(ctx, e.WorkflowOptions, e.WorkflowType, e.WorkflowInput) } else if sig := withStartAction.GetDoSignal(); sig != nil { @@ -38,6 +63,12 @@ func (e *ClientActionsExecutor) Start( } else { return fmt.Errorf("unsupported with_start_action: %v", withStartAction.String()) } + if err != nil && e.RecoverOriginalStart && withStartAction == nil { + var already *serviceerror.WorkflowExecutionAlreadyStarted + if errors.As(err, &already) && already.RunId != "" { + err = e.attachOriginalStart(ctx, already.RunId) + } + } if err != nil { return fmt.Errorf("failed to start kitchen sink workflow: %w", err) } @@ -170,17 +201,35 @@ func (e *ClientActionsExecutor) executeUpdateAction(ctx context.Context, upd *Do var handle client.WorkflowUpdateHandle if upd.WithStart { workflowOpts := e.WorkflowOptions - workflowOpts.WorkflowIDConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING + if !e.RecoverOriginalStart { + workflowOpts.WorkflowIDConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING + } else { + updateOpts.WaitForStage = client.WorkflowUpdateStageAccepted + } op := e.Client.NewWithStartWorkflowOperation(workflowOpts, e.WorkflowType, e.WorkflowInput) handle, err = e.Client.UpdateWithStartWorkflow(ctx, client.UpdateWithStartWorkflowOptions{ StartWorkflowOperation: op, UpdateOptions: updateOpts, }) + if err != nil && e.RecoverOriginalStart { + var already *serviceerror.WorkflowExecutionAlreadyStarted + if errors.As(err, &already) && already.RunId != "" { + if err := e.attachOriginalStart(ctx, already.RunId); err != nil { + return nil, err + } + // Retry the same update only on the history-verified original run. + updateOpts.RunID = already.RunId + handle, err = e.Client.UpdateWorkflow(ctx, updateOpts) + } + } } else { handle, err = e.Client.UpdateWorkflow(ctx, updateOpts) } if err == nil { + if upd.WithStart && e.RecoverOriginalStart { + e.originalStart.update = handle + } err = handle.Get(ctx, nil) if upd.WithStart { run = e.Client.GetWorkflow(ctx, handle.WorkflowID(), handle.RunID()) diff --git a/loadgen/kitchensink/original_start.go b/loadgen/kitchensink/original_start.go new file mode 100644 index 00000000..db3f9d5e --- /dev/null +++ b/loadgen/kitchensink/original_start.go @@ -0,0 +1,70 @@ +package kitchensink + +import ( + "context" + "errors" + "fmt" + + "github.com/google/uuid" + enumspb "go.temporal.io/api/enums/v1" + "go.temporal.io/sdk/client" + "go.temporal.io/sdk/converter" + "google.golang.org/protobuf/proto" +) + +const startIntentMemo = "omes.original-start" + +type originalStart struct { + intent string + action *WithStartClientAction + update client.WorkflowUpdateHandle +} + +func (e *ClientActionsExecutor) freezeStart(action *WithStartClientAction) { + e.originalStart = &originalStart{intent: uuid.NewString()} + memo := make(map[string]interface{}, len(e.WorkflowOptions.Memo)+1) + for k, v := range e.WorkflowOptions.Memo { + memo[k] = v + } + memo[startIntentMemo] = e.originalStart.intent + e.WorkflowOptions.Memo = memo + e.WorkflowOptions.WorkflowExecutionErrorWhenAlreadyStarted = true + e.WorkflowOptions.WorkflowIDReusePolicy = enumspb.WORKFLOW_ID_REUSE_POLICY_REJECT_DUPLICATE + e.WorkflowOptions.WorkflowIDConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_FAIL + if upd := action.GetDoUpdate(); upd != nil && upd.UpdateId == "" { + upd.UpdateId = uuid.NewString() + } + if action != nil { + e.originalStart.action = proto.Clone(action).(*WithStartClientAction) + } +} + +func (e *ClientActionsExecutor) attachOriginalStart(ctx context.Context, runID string) error { + if err := ctx.Err(); err != nil { + return err + } + history := e.Client.GetWorkflowHistory(ctx, e.WorkflowOptions.ID, runID, false, enumspb.HISTORY_EVENT_FILTER_TYPE_ALL_EVENT) + event, err := history.Next() + if err != nil { + return err + } + attrs := event.GetWorkflowExecutionStartedEventAttributes() + if attrs == nil { + return errors.New("original start history is missing its start event") + } + if attrs.GetMemo().GetFields()[startIntentMemo] == nil { + return errors.New("existing workflow has no original start identity") + } + var intent string + if err := converter.GetDefaultDataConverter().FromPayload(attrs.GetMemo().GetFields()[startIntentMemo], &intent); err != nil { + return fmt.Errorf("reading original start identity: %w", err) + } + if intent != e.originalStart.intent || + attrs.GetWorkflowType().GetName() != e.WorkflowType || + attrs.GetTaskQueue().GetName() != e.WorkflowOptions.TaskQueue { + return errors.New("existing workflow does not match the original start intent") + } + // Bind the history-verified run, never a later execution selected by workflow ID. + e.Handle = e.Client.GetWorkflow(ctx, e.WorkflowOptions.ID, runID) + return nil +} diff --git a/loadgen/kitchensink/original_start_test.go b/loadgen/kitchensink/original_start_test.go new file mode 100644 index 00000000..5e8ce249 --- /dev/null +++ b/loadgen/kitchensink/original_start_test.go @@ -0,0 +1,423 @@ +package kitchensink + +import ( + "context" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + "github.com/temporalio/omes/devserver" + "github.com/temporalio/omes/internal/versions" + enumspb "go.temporal.io/api/enums/v1" + "go.temporal.io/api/serviceerror" + "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/sdk/client" + "go.temporal.io/sdk/converter" + "go.temporal.io/sdk/temporal" + "go.temporal.io/sdk/worker" + "go.temporal.io/sdk/workflow" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type responseLossClient struct { + client.Client + starts int + multis int + original client.WorkflowRun + failMulti bool + failUpdate bool + loseMulti bool + loseOutcome bool + updateID string +} + +func (c *responseLossClient) ExecuteWorkflow(ctx context.Context, opts client.StartWorkflowOptions, wf interface{}, args ...interface{}) (client.WorkflowRun, error) { + c.starts++ + h, err := c.Client.ExecuteWorkflow(ctx, opts, wf, args...) + if err == nil && c.starts == 1 { + c.original = h + return nil, serviceerror.NewDeadlineExceeded("lost accepted start reply") + } + return h, err +} +func (c *responseLossClient) UpdateWithStartWorkflow(ctx context.Context, opts client.UpdateWithStartWorkflowOptions) (client.WorkflowUpdateHandle, error) { + c.multis++ + c.updateID = opts.UpdateOptions.UpdateID + if c.failMulti && c.multis == 1 { + return nil, serviceerror.NewUnavailable("start request not sent") + } + h, err := c.Client.UpdateWithStartWorkflow(ctx, opts) + if err != nil { + return nil, err + } + if c.loseMulti && c.multis == 1 { + return nil, serviceerror.NewDeadlineExceeded("lost multi-operation reply") + } + if c.loseOutcome { + return &outcomeLossHandle{WorkflowUpdateHandle: h}, nil + } + return h, nil +} + +func (c *responseLossClient) UpdateWorkflow(ctx context.Context, opts client.UpdateWorkflowOptions) (client.WorkflowUpdateHandle, error) { + if c.failUpdate { + c.failUpdate = false + return nil, serviceerror.NewUnavailable("update request not sent") + } + return c.Client.UpdateWorkflow(ctx, opts) +} + +type outcomeLossHandle struct { + client.WorkflowUpdateHandle + calls atomic.Int32 +} + +func (h *outcomeLossHandle) Get(ctx context.Context, v interface{}) error { + if h.calls.Add(1) == 1 { + return serviceerror.NewDeadlineExceeded("lost update outcome reply") + } + return h.WorkflowUpdateHandle.Get(ctx, v) +} + +func originalStartWorkflow(ctx workflow.Context, input *WorkflowInput) error { + updates := 0 + if err := workflow.SetQueryHandler(ctx, "updates", func() (int, error) { return updates, nil }); err != nil { + return err + } + if err := workflow.SetUpdateHandler(ctx, "do_actions_update", func(ctx workflow.Context, _ *DoActionsUpdate) error { + updates++ + if input.ExpectedSignalCount == 42 { + return temporal.NewApplicationError("update failed", "UpdateFailure") + } + return workflow.Sleep(ctx, 50*time.Millisecond) + }); err != nil { + return err + } + workflow.GetSignalChannel(ctx, "finish").Receive(ctx, nil) + if input.ExpectedSignalCount < 0 { + return temporal.NewNonRetryableApplicationError("original target failure", "TargetFailure", nil) + } + return nil +} + +func TestOriginalStartProtocol(t *testing.T) { + ref, err := versions.Get("SERVER_VERSION") + require.NoError(t, err) + server, err := devserver.Start(t.Context(), devserver.Options{Ref: ref}) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, server.Stop()) }) + ctx, cancel := context.WithTimeout(t.Context(), 4*time.Minute) + defer cancel() + c, err := client.DialContext(ctx, client.Options{HostPort: server.FrontendHostPort(), Namespace: "default"}) + require.NoError(t, err) + defer c.Close() + queue := "original-start-" + uuid.NewString() + w := worker.New(c, queue, worker.Options{}) + w.RegisterWorkflowWithOptions(originalStartWorkflow, workflow.RegisterOptions{Name: "kitchenSink"}) + require.NoError(t, w.Start()) + defer w.Stop() + newExecutor := func(cl client.Client) *ClientActionsExecutor { + e := &ClientActionsExecutor{ + Client: cl, + Namespace: "default", + WorkflowType: "kitchenSink", + WorkflowInput: &WorkflowInput{}, + WorkflowOptions: client.StartWorkflowOptions{ + ID: uuid.NewString(), + TaskQueue: queue, + WorkflowExecutionErrorWhenAlreadyStarted: true, + }, + } + e.RecoverOriginalStart = true + return e + } + for _, state := range []string{"running", "completed", "failed"} { + t.Run("lost_response_"+state, func(t *testing.T) { + lost := &responseLossClient{Client: c} + e := newExecutor(lost) + if state == "failed" { + e.WorkflowInput.ExpectedSignalCount = -1 + } + require.Error(t, e.Start(ctx, nil)) + original := lost.original + require.NotNil(t, original) + if state != "running" { + require.NoError(t, c.SignalWorkflow(ctx, original.GetID(), original.GetRunID(), "finish", nil)) + getErr := original.Get(ctx, nil) + if state == "failed" { + require.Error(t, getErr) + } else { + require.NoError(t, getErr) + } + } + require.NoError(t, e.Start(ctx, nil)) + require.Equal(t, original.GetRunID(), e.Handle.GetRunID(), "must never replace the original closed execution") + if state == "running" { + require.NoError(t, c.SignalWorkflow(ctx, original.GetID(), original.GetRunID(), "finish", nil)) + } + getErr := e.Handle.Get(ctx, nil) + if state == "failed" { + var failure *temporal.ApplicationError + require.ErrorAs(t, getErr, &failure) + require.Equal(t, "TargetFailure", failure.Type()) + require.NoError(t, e.Start(ctx, nil)) + require.Error(t, e.Handle.Get(ctx, nil)) + } else { + require.NoError(t, getErr) + } + }) + } + t.Run("different_input_converter", func(t *testing.T) { + binaryClient, err := client.DialContext(ctx, client.Options{ + HostPort: server.FrontendHostPort(), + Namespace: "default", + DataConverter: converter.NewCompositeDataConverter( + converter.NewProtoPayloadConverter(), + converter.NewJSONPayloadConverter(), + ), + }) + require.NoError(t, err) + defer binaryClient.Close() + lost := &responseLossClient{Client: binaryClient} + e := newExecutor(lost) + require.Error(t, e.Start(ctx, nil)) + require.NoError(t, e.Start(ctx, nil)) + require.Equal(t, lost.original.GetRunID(), e.Handle.GetRunID()) + require.NoError(t, c.SignalWorkflow(ctx, e.Handle.GetID(), e.Handle.GetRunID(), "finish", nil)) + require.NoError(t, e.Handle.Get(ctx, nil)) + }) + + t.Run("foreign_reuse", func(t *testing.T) { + lost := &responseLossClient{Client: c} + e := newExecutor(lost) + require.Error(t, e.Start(ctx, nil)) + original := lost.original + require.NoError(t, c.SignalWorkflow(ctx, original.GetID(), original.GetRunID(), "finish", nil)) + require.NoError(t, original.Get(ctx, nil)) + opts := e.WorkflowOptions + opts.Memo = map[string]interface{}{startIntentMemo: uuid.NewString()} + opts.WorkflowIDReusePolicy = enumspb.WORKFLOW_ID_REUSE_POLICY_ALLOW_DUPLICATE + foreign, err := c.ExecuteWorkflow(ctx, opts, "kitchenSink", &WorkflowInput{ExpectedSignalCount: 99}) + require.NoError(t, err) + require.Error(t, e.Start(ctx, nil)) + require.Nil(t, e.Handle) + require.NoError(t, c.SignalWorkflow(ctx, foreign.GetID(), foreign.GetRunID(), "finish", nil)) + require.NoError(t, foreign.Get(ctx, nil)) + }) + t.Run("foreign_existing", func(t *testing.T) { + e := newExecutor(c) + foreign, err := c.ExecuteWorkflow(ctx, e.WorkflowOptions, "kitchenSink", e.WorkflowInput) + require.NoError(t, err) + require.Error(t, e.Start(ctx, nil)) + require.Nil(t, e.Handle) + require.NoError(t, c.SignalWorkflow(ctx, foreign.GetID(), foreign.GetRunID(), "finish", nil)) + require.NoError(t, foreign.Get(ctx, nil)) + }) + t.Run("canceled", func(t *testing.T) { + e := newExecutor(c) + canceled, stop := context.WithCancel(ctx) + stop() + require.Error(t, e.Start(canceled, nil)) + require.Nil(t, e.Handle) + }) + action := func() *WithStartClientAction { + return &WithStartClientAction{ + Variant: &WithStartClientAction_DoUpdate{ + DoUpdate: &DoUpdate{ + WithStart: true, + Variant: &DoUpdate_DoActions{ + DoActions: &DoActionsUpdate{ + Variant: &DoActionsUpdate_DoActions{}, + }, + }, + }, + }, + } + } + for _, failure := range []string{"none", "update_unavailable"} { + t.Run("uws_already_started_"+failure, func(t *testing.T) { + observed := &responseLossClient{Client: c, failUpdate: failure == "update_unavailable"} + e := newExecutor(observed) + a := action() + e.freezeStart(a) + original, err := c.ExecuteWorkflow(ctx, e.WorkflowOptions, "kitchenSink", e.WorkflowInput) + require.NoError(t, err) + if failure == "update_unavailable" { + require.Error(t, e.Start(ctx, a)) + } + require.NoError(t, e.Start(ctx, a)) + require.Equal(t, original.GetRunID(), e.Handle.GetRunID()) + require.NoError(t, e.Start(ctx, a)) + value, err := c.QueryWorkflow(ctx, original.GetID(), original.GetRunID(), "updates") + require.NoError(t, err) + var updates int + require.NoError(t, value.Get(&updates)) + require.Equal(t, 1, updates) + require.NoError(t, c.SignalWorkflow(ctx, original.GetID(), original.GetRunID(), "finish", nil)) + require.NoError(t, original.Get(ctx, nil)) + }) + } + + t.Run("uws_confirmed_update_outcome", func(t *testing.T) { + lost := &responseLossClient{Client: c, loseOutcome: true} + e := newExecutor(lost) + a := action() + require.Error(t, e.Start(ctx, a)) + require.NoError(t, e.Start(ctx, a)) + require.Equal(t, 1, lost.multis) + require.NotEmpty(t, a.GetDoUpdate().GetUpdateId()) + require.Equal(t, lost.updateID, a.GetDoUpdate().GetUpdateId()) + require.NoError(t, c.SignalWorkflow(ctx, e.Handle.GetID(), e.Handle.GetRunID(), "finish", nil)) + require.NoError(t, e.Handle.Get(ctx, nil)) + }) + for _, failure := range []string{"before_send", "lost_reply"} { + t.Run("uws_retry_"+failure, func(t *testing.T) { + lost := &responseLossClient{ + Client: c, + failMulti: failure == "before_send", + loseMulti: failure == "lost_reply", + } + e := newExecutor(lost) + a := action() + require.Error(t, e.Start(ctx, a)) + updateID := lost.updateID + require.NotEmpty(t, updateID) + require.NoError(t, e.Start(ctx, a)) + require.Equal(t, 2, lost.multis) + require.Equal(t, updateID, lost.updateID) + value, err := c.QueryWorkflow(ctx, e.Handle.GetID(), e.Handle.GetRunID(), "updates") + require.NoError(t, err) + var updates int + require.NoError(t, value.Get(&updates)) + require.Equal(t, 1, updates, "retry must not execute the update twice") + canceled, stop := context.WithCancel(ctx) + stop() + require.ErrorIs(t, e.Start(canceled, a), context.Canceled) + require.NoError(t, c.SignalWorkflow(ctx, e.Handle.GetID(), e.Handle.GetRunID(), "finish", nil)) + require.NoError(t, e.Handle.Get(ctx, nil)) + }) + } + + t.Run("uws_closed_target_failure_is_not_replaced", func(t *testing.T) { + observed := &responseLossClient{Client: c} + e := newExecutor(observed) + e.WorkflowInput.ExpectedSignalCount = -1 + a := action() + require.NoError(t, e.Start(ctx, a)) + original := e.Handle.GetRunID() + require.NoError(t, c.SignalWorkflow(ctx, e.Handle.GetID(), original, "finish", nil)) + require.Error(t, e.Handle.Get(ctx, nil)) + require.NoError(t, e.Start(ctx, a)) + require.Equal(t, original, e.Handle.GetRunID()) + require.Equal(t, 1, observed.multis) + require.Error(t, e.Handle.Get(ctx, nil)) + }) + t.Run("uws_failed_update_outcome_is_preserved", func(t *testing.T) { + observed := &responseLossClient{Client: c} + e := newExecutor(observed) + e.WorkflowInput.ExpectedSignalCount = 42 + a := action() + require.Error(t, e.Start(ctx, a)) + require.Error(t, e.Start(ctx, a)) + require.Equal(t, 1, observed.multis) + require.NoError(t, c.SignalWorkflow(ctx, e.WorkflowOptions.ID, "", "finish", nil)) + require.NoError(t, c.GetWorkflow(ctx, e.WorkflowOptions.ID, "").Get(ctx, nil)) + }) + t.Run("uws_foreign_closed_failed_is_not_replaced", func(t *testing.T) { + e := newExecutor(c) + e.WorkflowInput.ExpectedSignalCount = -1 + foreign, err := c.ExecuteWorkflow(ctx, e.WorkflowOptions, "kitchenSink", e.WorkflowInput) + require.NoError(t, err) + require.NoError(t, c.SignalWorkflow(ctx, foreign.GetID(), foreign.GetRunID(), "finish", nil)) + require.Error(t, foreign.Get(ctx, nil)) + require.Error(t, e.Start(ctx, action())) + require.Nil(t, e.Handle) + }) + t.Run("uws_wire_reply_loss_and_external_reuse", func(t *testing.T) { + var lost atomic.Bool + var mu sync.Mutex + requests := map[string]bool{} + foreignRuns := make(chan client.WorkflowRun, 1) + intercept := func(callCtx context.Context, method string, req, reply interface{}, conn *grpc.ClientConn, invoke grpc.UnaryInvoker, opts ...grpc.CallOption) error { + if !strings.HasSuffix(method, "/ExecuteMultiOperation") { + return invoke(callCtx, method, req, reply, conn, opts...) + } + request := req.(*workflowservice.ExecuteMultiOperationRequest) + start := request.GetOperations()[0].GetStartWorkflow() + mu.Lock() + requests[start.GetRequestId()] = true + mu.Unlock() + err := invoke(callCtx, method, req, reply, conn, opts...) + if err == nil && !lost.Swap(true) { + originalRun := reply.(*workflowservice.ExecuteMultiOperationResponse).GetResponses()[0].GetStartWorkflow().GetRunId() + if err := c.SignalWorkflow(ctx, start.GetWorkflowId(), originalRun, "finish", nil); err != nil { + return err + } + if err := c.GetWorkflow(ctx, start.GetWorkflowId(), originalRun).Get(ctx, nil); err != nil { + return err + } + foreign, err := c.ExecuteWorkflow(ctx, client.StartWorkflowOptions{ + ID: start.GetWorkflowId(), + TaskQueue: queue, + WorkflowIDReusePolicy: enumspb.WORKFLOW_ID_REUSE_POLICY_ALLOW_DUPLICATE, + }, "kitchenSink", &WorkflowInput{ExpectedSignalCount: 99}) + if err != nil { + return err + } + foreignRuns <- foreign + return status.Error(codes.DeadlineExceeded, "lost real multi-operation reply after external ID reuse") + } + return err + } + wire, err := client.Dial(client.Options{ + HostPort: server.FrontendHostPort(), + Namespace: "default", + ConnectionOptions: client.ConnectionOptions{ + DialOptions: []grpc.DialOption{grpc.WithChainUnaryInterceptor(intercept)}, + }, + }) + require.NoError(t, err) + defer wire.Close() + e := newExecutor(wire) + a := action() + firstErr := e.Start(ctx, a) + secondErr := e.Start(ctx, a) + require.True(t, lost.Load(), "real successful multi-operation response must have been discarded") + var foreign client.WorkflowRun + select { + case foreign = <-foreignRuns: + case <-time.After(time.Second): + t.Fatal("external reuse was not established") + } + value, err := c.QueryWorkflow(ctx, foreign.GetID(), foreign.GetRunID(), "updates") + require.NoError(t, err) + var updates int + require.NoError(t, value.Get(&updates)) + require.Zero(t, updates, "recovery must not update a foreign replacement run") + mu.Lock() + requestCount := len(requests) + mu.Unlock() + require.Equal(t, 2, requestCount, "each outer attempt sends one multi-operation request") + require.Error(t, firstErr) + require.Error(t, secondErr) + require.NoError(t, c.SignalWorkflow(ctx, foreign.GetID(), foreign.GetRunID(), "finish", nil)) + require.NoError(t, foreign.Get(ctx, nil)) + }) + + t.Run("uws_foreign_is_not_updated", func(t *testing.T) { + e := newExecutor(c) + foreign, err := c.ExecuteWorkflow(ctx, e.WorkflowOptions, "kitchenSink", e.WorkflowInput) + require.NoError(t, err) + err = e.Start(ctx, action()) + require.Error(t, err) + require.Nil(t, e.Handle) + require.NoError(t, c.SignalWorkflow(ctx, foreign.GetID(), foreign.GetRunID(), "finish", nil)) + require.NoError(t, foreign.Get(ctx, nil)) + }) +} diff --git a/loadgen/scenario.go b/loadgen/scenario.go index 93396e21..16bb9796 100644 --- a/loadgen/scenario.go +++ b/loadgen/scenario.go @@ -396,6 +396,8 @@ type Run struct { // tracks how many attempts have been made for this iteration attemptCount int + + kitchenSinkExecution *kitchenSinkExecution } // NewRun creates a new run. @@ -529,17 +531,24 @@ type KitchenSinkWorkflowOptions struct { // completion ignoring its result. Concurrently it will perform any client actions specified in // kitchensink.TestInput.ClientSequence func (r *Run) ExecuteKitchenSinkWorkflow(ctx context.Context, options *KitchenSinkWorkflowOptions) error { - r.Logger.Debugf("Executing kitchen sink workflow with options: %v", options) - cancelCtx, cancel := context.WithCancel(ctx) - defer cancel() + return r.executeKitchenSinkWorkflow(ctx, options, r.newClientActionsExecutor(options)) +} - executor := &kitchensink.ClientActionsExecutor{ +func (r *Run) newClientActionsExecutor(options *KitchenSinkWorkflowOptions) *kitchensink.ClientActionsExecutor { + return &kitchensink.ClientActionsExecutor{ Client: r.Client, Namespace: r.Namespace, WorkflowOptions: options.StartOptions, WorkflowType: "kitchenSink", WorkflowInput: options.Params.GetWorkflowInput(), } +} + +func (r *Run) executeKitchenSinkWorkflow(ctx context.Context, options *KitchenSinkWorkflowOptions, executor *kitchensink.ClientActionsExecutor) error { + r.Logger.Debugf("Executing kitchen sink workflow with options: %v", options) + cancelCtx, cancel := context.WithCancel(ctx) + defer cancel() + startErr := executor.Start(ctx, options.Params.WithStartAction) if startErr != nil { return fmt.Errorf("failed to start kitchen sink workflow: %w", startErr) diff --git a/scenarios/throughput_start_compatibility_test.go b/scenarios/throughput_start_compatibility_test.go new file mode 100644 index 00000000..9b8c0d7a --- /dev/null +++ b/scenarios/throughput_start_compatibility_test.go @@ -0,0 +1,58 @@ +package scenarios + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/temporalio/omes/clioptions" + "github.com/temporalio/omes/internal/workertest" + "github.com/temporalio/omes/loadgen" + enumspb "go.temporal.io/api/enums/v1" +) + +func TestThroughputLegacyStartAndSnapshotResume(t *testing.T) { + env := workertest.SetupTestEnvironment(t, workertest.WithExecutorTimeout(time.Minute)) + id := fmt.Sprintf("legacy-start-%d", time.Now().UnixNano()) + info := loadgen.ScenarioInfo{ + RunID: id, + ExecutionID: id, + Configuration: loadgen.RunConfiguration{ + Iterations: 1, + IgnoreAlreadyStarted: true, + }, + Options: loadgen.MustResolveScenarioOptions("throughput_stress", map[string]string{ + IterFlag: "1", + ContinueAsNewAfterIterFlag: "0", + NexusEnabledFlag: "false", + SleepTimeFlag: "1ms", + VisibilityVerificationTimeoutFlag: "10s", + }), + } + first := newThroughputStressExecutor() + + _, err := env.RunExecutorTest(t, first, info, clioptions.LangGo) + require.NoError(t, err) + + workflowID := fmt.Sprintf("w-%s-%s-1", id, id) + checkUnmarked := func() { + history := env.TemporalClient().GetWorkflowHistory(t.Context(), workflowID, "", false, enumspb.HISTORY_EVENT_FILTER_TYPE_ALL_EVENT) + event, err := history.Next() + require.NoError(t, err) + require.NotContains(t, event.GetWorkflowExecutionStartedEventAttributes().GetMemo().GetFields(), "omes.original-start") + } + checkUnmarked() + + resumed := newThroughputStressExecutor() + require.NoError(t, resumed.LoadState(func(v any) error { + v.(*tpsState).CompletedIterations = 0 + return nil + })) + info.Configuration.IgnoreAlreadyStarted = false + + _, err = env.RunExecutorTest(t, resumed, info, clioptions.LangGo) + require.NoError(t, err) + require.Equal(t, 1, resumed.Snapshot().(tpsState).CompletedIterations) + checkUnmarked() +} diff --git a/scenarios/throughput_stress.go b/scenarios/throughput_stress.go index 3fa983e9..4b2d5145 100644 --- a/scenarios/throughput_stress.go +++ b/scenarios/throughput_stress.go @@ -392,6 +392,7 @@ func (t *tpsExecutor) Run(ctx context.Context, info loadgen.ScenarioInfo) error mainResult.finishedAt = time.Now() } else { ksExec := &loadgen.KitchenSinkExecutor{ + RecoverOriginalStart: !isResuming && !info.Configuration.IgnoreAlreadyStarted, TestInput: &TestInput{ WorkflowInput: &WorkflowInput{ InitialActions: []*ActionSet{},