From 03269a30d262e3a6f446b0f2ba6a8a276a651f78 Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Fri, 6 Nov 2026 14:48:16 -0800 Subject: [PATCH 01/21] Add executor support for extended orchestration sessions Split OrchestrationContext.start into begin and run so an orchestration can stay loaded between turns. Add the optional SessionExecutor and OrchestratorSession contract: Commit retires the actions that the backend has applied, Resume runs a batch of new events on the live state, and Close unloads with the existing teardown semantics. History length now uses a history base, so a resumed turn reports the same length and history limit as a full replay. A turn that completes, continues as new, hits a limit, panics, or processes only part of its events never keeps a session. Tests compare every session turn with a fresh full replay of the same history across activities, timers, sub-orchestrations, events, coroutines, cancellation, entities, critical sections, and suspension. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- task/executor.go | 81 ++- task/executor_contract.go | 47 ++ task/orchestrator.go | 94 +++- task/session.go | 126 +++++ task/session_test.go | 1070 +++++++++++++++++++++++++++++++++++++ 5 files changed, 1385 insertions(+), 33 deletions(-) create mode 100644 task/session.go create mode 100644 task/session_test.go diff --git a/task/executor.go b/task/executor.go index 95480291..903e148c 100644 --- a/task/executor.go +++ b/task/executor.go @@ -249,12 +249,68 @@ func (te *taskExecutor) ExecuteOrchestrator( newEvents []*protos.HistoryEvent, entityParameters *protos.OrchestratorEntityParameters, ) (*ExecutionResults, error) { + results, orchestrationCtx, err := te.prepareOrchestrator(id, oldEvents, newEvents, entityParameters) + if results != nil || err != nil { + return results, err + } + actions := orchestrationCtx.start() + return te.orchestratorResults(orchestrationCtx, actions), nil +} + +// StartOrchestratorSession implements SessionExecutor. It runs the first turn +// like ExecuteOrchestrator, but keeps the orchestration loaded when the turn +// can be continued with new events. +func (te *taskExecutor) StartOrchestratorSession( + _ context.Context, + id api.InstanceID, + oldEvents []*protos.HistoryEvent, + newEvents []*protos.HistoryEvent, + entityParameters *protos.OrchestratorEntityParameters, +) (*ExecutionResults, OrchestratorSession, error) { + results, orchestrationCtx, err := te.prepareOrchestrator(id, oldEvents, newEvents, entityParameters) + if results != nil || err != nil { + return results, nil, err + } + session := &orchestratorSession{executor: te, ctx: orchestrationCtx} + orchestrationCtx.begin() + return session.finishTurn(orchestrationCtx.run()) +} + +// orchestratorResults builds the response for one completed turn. +func (te *taskExecutor) orchestratorResults( + orchestrationCtx *OrchestrationContext, + actions []*protos.OrchestratorAction, +) *ExecutionResults { + te.reportHistoryMetric(orchestrationCtx) + + results := &ExecutionResults{ + Response: &protos.OrchestratorResponse{ + InstanceId: string(orchestrationCtx.ID), + Actions: actions, + CustomStatus: wrapperspb.String(orchestrationCtx.customStatus), + }, + } + if orchestrationCtx.maxEventsPerTurnExceeded && !orchestrationCtx.hasCompletionAction() { + results.Response.NumEventsProcessed = wrapperspb.Int32(int32(orchestrationCtx.processedEventsThisTurn)) + } + return results +} + +// prepareOrchestrator checks the work item before any orchestrator code runs. +// It returns either a final result or error, or a new orchestration context. +func (te *taskExecutor) prepareOrchestrator( + id api.InstanceID, + oldEvents []*protos.HistoryEvent, + newEvents []*protos.HistoryEvent, + entityParameters *protos.OrchestratorEntityParameters, +) (*ExecutionResults, *OrchestrationContext, error) { entitiesSupported, err := validateEntityParameters(entityParameters) if err != nil { - return nil, err + return nil, nil, err } if isRewindRequest(oldEvents, newEvents) { - return buildRewindResult(id, oldEvents, newEvents) + results, err := buildRewindResult(id, oldEvents, newEvents) + return results, nil, err } started := startedEvent(oldEvents, newEvents) name := started.GetName() @@ -262,7 +318,7 @@ func (te *taskExecutor) ExecuteOrchestrator( if versionErr := te.versioning.check(version); versionErr != nil && !allowsUnversionedTask(te.unversionedOrchestrators, name, version) { if te.versioning.FailureStrategy == VersionFailureReject { - return nil, versionErr + return nil, nil, versionErr } return &ExecutionResults{ Response: &protos.OrchestratorResponse{ @@ -277,11 +333,11 @@ func (te *taskExecutor) ExecuteOrchestrator( ), }, }, - }, nil + }, nil, nil } if te.orchestratorNotFound == OrchestratorNotFoundReject && name != "" { if !te.Registry.hasOrchestrator(name, version) { - return nil, newTaskNotRegisteredError( + return nil, nil, newTaskNotRegisteredError( orchestratorTaskNotFoundErrorType, name, version, @@ -301,20 +357,7 @@ func (te *taskExecutor) ExecuteOrchestrator( te.converter, entitiesSupported, ) - actions := orchestrationCtx.start() - te.reportHistoryMetric(orchestrationCtx) - - results := &ExecutionResults{ - Response: &protos.OrchestratorResponse{ - InstanceId: string(id), - Actions: actions, - CustomStatus: wrapperspb.String(orchestrationCtx.customStatus), - }, - } - if orchestrationCtx.maxEventsPerTurnExceeded && !orchestrationCtx.hasCompletionAction() { - results.Response.NumEventsProcessed = wrapperspb.Int32(int32(orchestrationCtx.processedEventsThisTurn)) - } - return results, nil + return nil, orchestrationCtx, nil } func validateEntityParameters(parameters *protos.OrchestratorEntityParameters) (bool, error) { diff --git a/task/executor_contract.go b/task/executor_contract.go index 1d5c3d69..673a78ab 100644 --- a/task/executor_contract.go +++ b/task/executor_contract.go @@ -36,3 +36,50 @@ type Executor interface { type EntityExecutor interface { ExecuteEntity(context.Context, *protos.EntityBatchRequest) (*protos.EntityBatchResult, error) } + +// SessionExecutor is an optional extension of [Executor] for extended +// sessions. An extended session keeps an orchestration loaded in memory +// between turns, so the next turn only needs the new events instead of a +// replay of the full history. Like [Executor], it is an internal contract +// between the task runtime and the gRPC worker. +type SessionExecutor interface { + // StartOrchestratorSession runs one turn exactly like + // [Executor.ExecuteOrchestrator]. It also returns a session when the + // orchestration can continue in memory. A nil session means that the + // caller must complete the turn without a session. + StartOrchestratorSession( + ctx context.Context, + iid api.InstanceID, + oldEvents []*protos.HistoryEvent, + newEvents []*protos.HistoryEvent, + entityParameters *protos.OrchestratorEntityParameters, + ) (*ExecutionResults, OrchestratorSession, error) +} + +// OrchestratorSession is an orchestration that stays loaded between turns. +// A session has one owner and is not safe for concurrent use. +// +// After each turn that returns a session, the owner must call Commit once +// the backend has accepted the turn's actions, and then either Resume or +// Close. +type OrchestratorSession interface { + // Commit marks the actions that the backend has durably applied. They + // are not sent again, in the same way that replayed history retires + // them. Pass exactly the actions that were sent for the last turn. + Commit(sent []*protos.OrchestratorAction) + // Resume runs the next turn with a batch of new events. The batch must + // start with an OrchestratorStarted event. historyBase is the number of + // history events that the backend has stored for the current execution. + // The returned session is nil when the orchestration can no longer stay + // loaded; the session is then already closed. On error the session is + // also closed and the caller must fall back to a full replay. + Resume( + ctx context.Context, + batch []*protos.HistoryEvent, + historyBase int, + ) (*ExecutionResults, OrchestratorSession, error) + // Close unloads the orchestration. Deferred orchestrator code that runs + // during the unload cannot schedule durable work. Close is safe to call + // more than once. + Close() +} diff --git a/task/orchestrator.go b/task/orchestrator.go index 63f7b9fe..b0f033c7 100644 --- a/task/orchestrator.go +++ b/task/orchestrator.go @@ -57,11 +57,15 @@ type OrchestrationContext struct { rawInput []byte oldEvents []*protos.HistoryEvent newEvents []*protos.HistoryEvent - suspendedEvents []replayEvent - resumedEvents []replayEvent - isSuspended bool - isTerminated bool - historyIndex int + // historyBase is the number of stored history events before newEvents. It + // equals len(oldEvents) for a normal turn. A resumed session turn has no + // oldEvents, so it uses the stored event count reported by the backend. + historyBase int + suspendedEvents []replayEvent + resumedEvents []replayEvent + isSuspended bool + isTerminated bool + historyIndex int // processedEventsThisTurn follows DTS numEventsProcessed semantics and // excludes orchestration control markers. processedEventsThisTurn int @@ -285,17 +289,22 @@ func newOrchestrationContext( registry: registry, oldEvents: oldEvents, newEvents: newEvents, + historyBase: len(oldEvents), bufferedExternalEvents: make(map[string]*list.List), pendingExternalEventTasks: make(map[string]*list.List), eventChannels: make(map[string]any), eventWaiters: make(map[string]map[*coroutine]struct{}), } - ctx.maxHistoryEventsExceeded = options.MaxHistoryEvents > 0 && - len(oldEvents)+len(newEvents) > options.MaxHistoryEvents + ctx.maxHistoryEventsExceeded = ctx.historyLimitReached() ctx.scope = newCancellationScope(nil) return ctx } +func (ctx *OrchestrationContext) historyLimitReached() bool { + limit := ctx.orchestrationOptions.MaxHistoryEvents + return limit > 0 && ctx.historyBase+len(ctx.newEvents) > limit +} + func (ctx *OrchestrationContext) engineContext() *OrchestrationContext { if ctx.root != nil { return ctx.root @@ -367,7 +376,15 @@ func (ctx *OrchestrationContext) WithCancel() (*OrchestrationContext, func()) { return child, cancel } -func (ctx *OrchestrationContext) start() (actions []*protos.OrchestratorAction) { +// start runs one complete turn and then unloads the orchestration. +func (ctx *OrchestrationContext) start() []*protos.OrchestratorAction { + ctx.begin() + defer ctx.unload() + return ctx.run() +} + +// begin resets the per-execution state and creates the coroutine scheduler. +func (ctx *OrchestrationContext) begin() { ctx.historyIndex = 0 ctx.processedEventsThisTurn = 0 ctx.sequenceNumber = 0 @@ -378,10 +395,59 @@ func (ctx *OrchestrationContext) start() (actions []*protos.OrchestratorAction) ctx.pendingEntityTasks = make(map[string]*completableTask) ctx.clearCriticalSection() ctx.scheduler = newCoroutineScheduler(ctx) - defer func() { - ctx.scheduler.shutdown() - ctx.scheduler = nil - }() +} + +// unload stops all coroutines with the forced-unload (teardown) semantics. +// It is safe to call more than once. +func (ctx *OrchestrationContext) unload() { + if ctx.scheduler == nil { + return + } + ctx.scheduler.shutdown() + ctx.scheduler = nil +} + +// commit retires the actions that the backend has durably applied. This has +// the same effect as replaying the history events that those actions create. +func (ctx *OrchestrationContext) commit(sent []*protos.OrchestratorAction) { + for _, action := range sent { + if action == nil { + continue + } + if _, ok := ctx.pendingActions[action.Id]; !ok { + continue + } + if requested := action.GetSendEntityMessage().GetEntityLockRequested(); requested != nil && + ctx.criticalSectionID != "" && requested.CriticalSectionId == ctx.criticalSectionID { + ctx.criticalSectionRequestCommitted = true + } + delete(ctx.pendingActions, action.Id) + } +} + +// resume runs one more turn on the loaded orchestration. The batch contains +// only new events, and historyBase is the number of events that the backend +// has already stored for this execution. +func (ctx *OrchestrationContext) resume(batch []*protos.HistoryEvent, historyBase int) []*protos.OrchestratorAction { + ctx.oldEvents = nil + ctx.newEvents = batch + ctx.historyBase = historyBase + ctx.historyIndex = 0 + ctx.IsReplaying = false + ctx.processedEventsThisTurn = 0 + ctx.maxEventsPerTurnExceeded = false + ctx.maxHistoryEventsExceeded = ctx.historyLimitReached() + // A full replay would see events buffered by an earlier suspension as + // old history, so they are replayed when the resume drains them. + for i := range ctx.suspendedEvents { + ctx.suspendedEvents[i].isReplaying = true + } + return ctx.run() +} + +// run processes the remaining events and runs coroutines until no more +// progress is possible in this turn. It returns the pending actions. +func (ctx *OrchestrationContext) run() (actions []*protos.OrchestratorAction) { defer func() { if value := recover(); value != nil { ctx.continuedAsNew = false @@ -522,10 +588,10 @@ func countsTowardTurnBudget(event *protos.HistoryEvent) bool { event.GetExecutionResumed() == nil } -// HistoryLength returns the total old and new history supplied to this execution. +// HistoryLength returns the total old and new history of this execution turn. func (ctx *OrchestrationContext) HistoryLength() int { engine := ctx.engineContext() - return len(engine.oldEvents) + len(engine.newEvents) + return engine.historyBase + len(engine.newEvents) } // HistoryLimitExceeded reports whether MaxHistoryEvents was exceeded. diff --git a/task/session.go b/task/session.go new file mode 100644 index 00000000..da3dee65 --- /dev/null +++ b/task/session.go @@ -0,0 +1,126 @@ +package task + +import ( + "context" + "errors" + "fmt" + + "github.com/microsoft/durabletask-go/internal/protos" +) + +var _ SessionExecutor = (*taskExecutor)(nil) + +type orchestratorSessionState int + +const ( + sessionAwaitingCommit orchestratorSessionState = iota + sessionAwaitingResume + sessionClosed +) + +var errOrchestratorSessionClosed = errors.New("orchestrator session is closed") + +// orchestratorSession keeps one OrchestrationContext and its coroutines +// loaded between turns. +type orchestratorSession struct { + executor *taskExecutor + ctx *OrchestrationContext + state orchestratorSessionState + // closeErr explains why the session was closed by a misuse, if it was. + closeErr error +} + +func (s *orchestratorSession) Commit(sent []*protos.OrchestratorAction) { + if s.state != sessionAwaitingCommit { + s.closeWith(fmt.Errorf("orchestrator session commit is out of order")) + return + } + s.ctx.commit(sent) + s.state = sessionAwaitingResume +} + +func (s *orchestratorSession) Resume( + _ context.Context, + batch []*protos.HistoryEvent, + historyBase int, +) (*ExecutionResults, OrchestratorSession, error) { + switch s.state { + case sessionClosed: + if s.closeErr != nil { + return nil, nil, s.closeErr + } + return nil, nil, errOrchestratorSessionClosed + case sessionAwaitingCommit: + err := fmt.Errorf("orchestrator session resumed before the last turn was committed") + s.closeWith(err) + return nil, nil, err + } + if err := validateSessionBatch(batch, historyBase); err != nil { + s.closeWith(err) + return nil, nil, err + } + return s.finishTurn(s.ctx.resume(batch, historyBase)) +} + +func (s *orchestratorSession) Close() { + s.closeWith(nil) +} + +func (s *orchestratorSession) closeWith(err error) { + if s.state == sessionClosed { + return + } + s.state = sessionClosed + s.closeErr = err + s.ctx.unload() +} + +// finishTurn builds the turn result and decides whether the orchestration can +// stay loaded. Only a turn that ends with live, waiting orchestrator code and +// no unprocessed events can continue with new events. +func (s *orchestratorSession) finishTurn( + actions []*protos.OrchestratorAction, +) (*ExecutionResults, OrchestratorSession, error) { + results := s.executor.orchestratorResults(s.ctx, actions) + if !s.ctx.canContinueSession() { + s.Close() + return results, nil, nil + } + s.state = sessionAwaitingCommit + return results, s, nil +} + +// canContinueSession reports whether the orchestration can stay loaded after +// the current turn. +func (ctx *OrchestrationContext) canContinueSession() bool { + scheduler := ctx.scheduler + return scheduler != nil && + scheduler.root != nil && + !scheduler.isStopping() && + !ctx.isTerminated && + !ctx.hasCompletionAction() && + // The backend sends unprocessed events again in the next work item, so + // a partial turn must not keep them in memory. + !ctx.maxEventsPerTurnExceeded && + !ctx.maxHistoryEventsExceeded +} + +func validateSessionBatch(batch []*protos.HistoryEvent, historyBase int) error { + if historyBase < 0 { + return fmt.Errorf("orchestrator session history base must not be negative: %d", historyBase) + } + if len(batch) == 0 || batch[0].GetOrchestratorStarted() == nil { + return fmt.Errorf("orchestrator session batch must start with an OrchestratorStarted event") + } + for _, event := range batch { + switch { + case event == nil: + return fmt.Errorf("orchestrator session batch must not contain nil events") + case event.GetExecutionStarted() != nil: + return fmt.Errorf("orchestrator session batch must not start a new execution") + case event.GetExecutionRewound() != nil: + return fmt.Errorf("orchestrator session batch must not contain a rewind request") + } + } + return nil +} diff --git a/task/session_test.go b/task/session_test.go new file mode 100644 index 00000000..70cdd927 --- /dev/null +++ b/task/session_test.go @@ -0,0 +1,1070 @@ +package task + +import ( + "context" + "errors" + "fmt" + "runtime" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" + "google.golang.org/protobuf/types/known/wrapperspb" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/helpers" + "github.com/microsoft/durabletask-go/internal/protos" +) + +var sessionTestStart = time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + +// sessionScenario describes a multi-turn orchestration run against a fake +// backend. After each turn the backend stores the batch plus the history +// events created from the sent actions, then sends a new batch. +type sessionScenario struct { + orchestrator Orchestrator + input *wrapperspb.StringValue + options []TaskExecutorOption + // perTurn limits how many outstanding responses are delivered in one + // batch. Zero delivers all of them. + perTurn int + // extra adds events to the batch of the given resumed turn (1-based). + extra map[int][]*protos.HistoryEvent + // respond overrides the default response for an action. Return nil to + // use the default. + respond func(action *protos.OrchestratorAction) *protos.HistoryEvent + // step is the time between turns. Zero means one minute. + step time.Duration +} + +type sessionRun struct { + responses []*protos.OrchestratorResponse + // historyLengths are the HistoryLength values reported by the session path. + historyLengths []int + sessionEnded bool +} + +const sessionTestInstance = api.InstanceID("session-instance") + +func sessionOrchestratorStarted(at time.Time) *protos.HistoryEvent { + return &protos.HistoryEvent{ + EventId: -1, + Timestamp: timestamppb.New(at), + EventType: &protos.HistoryEvent_OrchestratorStarted{OrchestratorStarted: &protos.OrchestratorStartedEvent{}}, + } +} + +func sessionExecutionStarted(name string, input *wrapperspb.StringValue) *protos.HistoryEvent { + event := helpers.NewExecutionStartedEvent(name, string(sessionTestInstance), input, nil, nil, nil) + event.Timestamp = timestamppb.New(sessionTestStart) + event.GetExecutionStarted().OrchestrationInstance.ExecutionId = wrapperspb.String("execution-1") + return event +} + +func newSessionTestExecutor( + registry *TaskRegistry, + lengths *[]int, + options ...TaskExecutorOption, +) *taskExecutor { + hooks := MetricsHooks{History: func(metric HistoryMetric) { + *lengths = append(*lengths, metric.HistoryLength) + }} + all := append([]TaskExecutorOption{WithMetricsHooks(hooks)}, options...) + return NewTaskExecutor(registry, all...).(*taskExecutor) +} + +// historyFromActions builds the history events that the backend stores for +// the actions sent in one turn. +func historyFromActions(t *testing.T, actions []*protos.OrchestratorAction) []*protos.HistoryEvent { + t.Helper() + events := make([]*protos.HistoryEvent, 0, len(actions)) + for _, action := range actions { + action = proto.Clone(action).(*protos.OrchestratorAction) + event := &protos.HistoryEvent{EventId: action.Id, Timestamp: timestamppb.New(sessionTestStart)} + switch { + case action.GetScheduleTask() != nil: + scheduled := action.GetScheduleTask() + event.EventType = &protos.HistoryEvent_TaskScheduled{TaskScheduled: &protos.TaskScheduledEvent{ + Name: scheduled.Name, Version: scheduled.Version, Input: scheduled.Input, Tags: scheduled.Tags, + }} + case action.GetCreateTimer() != nil: + event.EventType = &protos.HistoryEvent_TimerCreated{TimerCreated: &protos.TimerCreatedEvent{ + FireAt: action.GetCreateTimer().FireAt, + }} + case action.GetCreateSubOrchestration() != nil: + created := action.GetCreateSubOrchestration() + event.EventType = &protos.HistoryEvent_SubOrchestrationInstanceCreated{ + SubOrchestrationInstanceCreated: &protos.SubOrchestrationInstanceCreatedEvent{ + InstanceId: created.InstanceId, Name: created.Name, Version: created.Version, Input: created.Input, + }, + } + case action.GetSendEvent() != nil: + sent := action.GetSendEvent() + event.EventType = &protos.HistoryEvent_EventSent{EventSent: &protos.EventSentEvent{ + InstanceId: sent.GetInstance().GetInstanceId(), Name: sent.Name, Input: sent.Data, + }} + case action.GetSendEntityMessage() != nil: + message := action.GetSendEntityMessage() + switch { + case message.GetEntityOperationCalled() != nil: + event.EventType = &protos.HistoryEvent_EntityOperationCalled{EntityOperationCalled: message.GetEntityOperationCalled()} + case message.GetEntityOperationSignaled() != nil: + event.EventType = &protos.HistoryEvent_EntityOperationSignaled{EntityOperationSignaled: message.GetEntityOperationSignaled()} + case message.GetEntityLockRequested() != nil: + event.EventType = &protos.HistoryEvent_EntityLockRequested{EntityLockRequested: message.GetEntityLockRequested()} + case message.GetEntityUnlockSent() != nil: + event.EventType = &protos.HistoryEvent_EntityUnlockSent{EntityUnlockSent: message.GetEntityUnlockSent()} + default: + t.Fatalf("unsupported entity message: %v", message) + } + case action.GetCompleteOrchestration() != nil: + continue + default: + t.Fatalf("unsupported action: %v", action) + } + events = append(events, event) + } + return events +} + +// defaultResponse returns the inbox event that answers an action, or nil if +// the action has no response. +func defaultResponse(action *protos.OrchestratorAction) *protos.HistoryEvent { + event := &protos.HistoryEvent{EventId: -1, Timestamp: timestamppb.New(sessionTestStart)} + switch { + case action.GetScheduleTask() != nil: + result := wrapperspb.String(fmt.Sprintf("%q", fmt.Sprintf("%s-%d", action.GetScheduleTask().Name, action.Id))) + event.EventType = &protos.HistoryEvent_TaskCompleted{TaskCompleted: &protos.TaskCompletedEvent{ + TaskScheduledId: action.Id, Result: result, + }} + case action.GetCreateTimer() != nil: + event.EventType = &protos.HistoryEvent_TimerFired{TimerFired: &protos.TimerFiredEvent{ + TimerId: action.Id, FireAt: action.GetCreateTimer().FireAt, + }} + case action.GetCreateSubOrchestration() != nil: + event.EventType = &protos.HistoryEvent_SubOrchestrationInstanceCompleted{ + SubOrchestrationInstanceCompleted: &protos.SubOrchestrationInstanceCompletedEvent{ + TaskScheduledId: action.Id, Result: wrapperspb.String(`"child-done"`), + }, + } + case action.GetSendEntityMessage().GetEntityOperationCalled() != nil: + called := action.GetSendEntityMessage().GetEntityOperationCalled() + event.EventType = &protos.HistoryEvent_EntityOperationCompleted{ + EntityOperationCompleted: &protos.EntityOperationCompletedEvent{ + RequestId: called.RequestId, Output: wrapperspb.String(fmt.Sprintf("%q", called.Operation+"-result")), + }, + } + case action.GetSendEntityMessage().GetEntityLockRequested() != nil: + requested := action.GetSendEntityMessage().GetEntityLockRequested() + event.EventType = &protos.HistoryEvent_EntityLockGranted{ + EntityLockGranted: &protos.EntityLockGrantedEvent{CriticalSectionId: requested.CriticalSectionId}, + } + default: + return nil + } + return event +} + +// runSessionEquivalence runs the scenario through one session and, at every +// turn, through a fresh full replay of the same stored history. Both paths +// must produce the same response and the same history length. +func runSessionEquivalence(t *testing.T, scenario sessionScenario) sessionRun { + t.Helper() + baseline := runtime.NumGoroutine() + registry := NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("session", scenario.orchestrator)) + var sessionLengths, replayLengths []int + sessionExecutor := newSessionTestExecutor(registry, &sessionLengths, scenario.options...) + replayExecutor := newSessionTestExecutor(registry, &replayLengths, scenario.options...) + step := scenario.step + if step == 0 { + step = time.Minute + } + respond := func(action *protos.OrchestratorAction) *protos.HistoryEvent { + if scenario.respond != nil { + if event := scenario.respond(action); event != nil { + return event + } + } + return defaultResponse(action) + } + lastExtraTurn := 0 + for turn := range scenario.extra { + lastExtraTurn = max(lastExtraTurn, turn) + } + + ctx := context.Background() + params := supportedEntityParameters() + var stored, outstanding []*protos.HistoryEvent + batch := []*protos.HistoryEvent{ + sessionOrchestratorStarted(sessionTestStart), + sessionExecutionStarted("session", scenario.input), + } + result, session, err := sessionExecutor.StartOrchestratorSession(ctx, sessionTestInstance, nil, batch, params) + require.NoError(t, err) + run := sessionRun{} + for turn := 0; ; turn++ { + replay, replayErr := replayExecutor.ExecuteOrchestrator(ctx, sessionTestInstance, stored, batch, params) + require.NoError(t, replayErr) + requireSameResponse(t, turn, replay.Response, result.Response) + require.Equal(t, replayLengths[len(replayLengths)-1], sessionLengths[len(sessionLengths)-1], "turn %d history length", turn) + require.Equal(t, len(stored)+len(batch), sessionLengths[len(sessionLengths)-1], "turn %d history length", turn) + run.responses = append(run.responses, result.Response) + run.historyLengths = append(run.historyLengths, sessionLengths[len(sessionLengths)-1]) + + if session == nil { + run.sessionEnded = true + break + } + require.Nil(t, result.Response.NumEventsProcessed, "turn %d: a partial turn must end the session", turn) + require.Nil(t, completionOf(result.Response), "turn %d: a completed turn must end the session", turn) + + // The backend applies the turn and acknowledges it. + stored = append(stored, batch...) + stored = append(stored, historyFromActions(t, result.Response.Actions)...) + session.Commit(result.Response.Actions) + for _, action := range result.Response.Actions { + if response := respond(action); response != nil { + outstanding = append(outstanding, response) + } + } + + next := turn + 1 + if len(outstanding) == 0 && next > lastExtraTurn { + session.Close() + break + } + require.Less(t, next, 50, "scenario does not finish") + deliver := len(outstanding) + if scenario.perTurn > 0 { + deliver = min(deliver, scenario.perTurn) + } + batch = []*protos.HistoryEvent{sessionOrchestratorStarted(sessionTestStart.Add(time.Duration(next) * step))} + batch = append(batch, outstanding[:deliver]...) + outstanding = outstanding[deliver:] + batch = append(batch, scenario.extra[next]...) + + result, session, err = session.Resume(ctx, batch, len(stored)) + require.NoError(t, err) + } + requireNoGoroutineLeak(t, baseline) + return run +} + +func requireSameResponse(t *testing.T, turn int, want, got *protos.OrchestratorResponse) { + t.Helper() + if !proto.Equal(want, got) { + t.Fatalf("turn %d: session response differs from full replay\nreplay: %v\nsession: %v", turn, want, got) + } +} + +func completionOf(response *protos.OrchestratorResponse) *protos.CompleteOrchestrationAction { + for _, action := range response.GetActions() { + if completed := action.GetCompleteOrchestration(); completed != nil { + return completed + } + } + return nil +} + +func requireNoGoroutineLeak(t *testing.T, baseline int) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for runtime.NumGoroutine() > baseline && time.Now().Before(deadline) { + runtime.Gosched() + time.Sleep(5 * time.Millisecond) + } + require.LessOrEqual(t, runtime.NumGoroutine(), baseline, "orchestration goroutines leaked") +} + +func requireCompletedWith(t *testing.T, run sessionRun, output string) { + t.Helper() + require.True(t, run.sessionEnded, "session should end with the orchestration") + completed := completionOf(run.responses[len(run.responses)-1]) + require.NotNil(t, completed) + require.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, completed.OrchestrationStatus, + "failure: %v", completed.GetFailureDetails()) + require.Equal(t, output, completed.GetResult().GetValue()) +} + +func eventRaised(name, rawInput string) *protos.HistoryEvent { + event := helpers.NewEventRaisedEvent(name, wrapperspb.String(rawInput)) + event.Timestamp = timestamppb.New(sessionTestStart) + return event +} + +func suspended() *protos.HistoryEvent { + event := helpers.NewSuspendOrchestrationEvent("pause") + event.Timestamp = timestamppb.New(sessionTestStart) + return event +} + +func resumed() *protos.HistoryEvent { + event := helpers.NewResumeOrchestrationEvent("continue") + event.Timestamp = timestamppb.New(sessionTestStart) + return event +} + +func TestSessionReplayEquivalence(t *testing.T) { + counter := api.NewEntityID("counter", "a") + other := api.NewEntityID("counter", "b") + + tests := []struct { + name string + scenario sessionScenario + output string + check func(t *testing.T, run sessionRun) + }{ + { + name: "activities", + output: `["first-0","second-1","third-2"]`, + scenario: sessionScenario{orchestrator: func(ctx *OrchestrationContext) (any, error) { + results := make([]string, 0, 3) + for _, name := range []string{"first", "second", "third"} { + var result string + if err := ctx.CallActivity(name, WithActivityInput(len(results))).Await(&result); err != nil { + return nil, err + } + results = append(results, result) + } + return results, nil + }}, + }, + { + name: "activity retry", + output: `"flaky-2"`, + scenario: func() sessionScenario { + failed := false + return sessionScenario{ + orchestrator: func(ctx *OrchestrationContext) (any, error) { + var result string + err := ctx.CallActivity("flaky", WithActivityRetryPolicy(&RetryPolicy{ + MaxAttempts: 3, + InitialRetryInterval: 10 * time.Second, + })).Await(&result) + return result, err + }, + respond: func(action *protos.OrchestratorAction) *protos.HistoryEvent { + if action.GetScheduleTask() == nil || failed { + return nil + } + failed = true + event := helpers.NewTaskFailedEvent(action.Id, &protos.TaskFailureDetails{ + ErrorType: "Transient", ErrorMessage: "try again", + }) + event.Timestamp = timestamppb.New(sessionTestStart) + return event + }, + } + }(), + }, + { + name: "timers and long timer chunks", + output: `"done"`, + scenario: sessionScenario{ + options: []TaskExecutorOption{WithMaximumTimerInterval(time.Minute)}, + orchestrator: func(ctx *OrchestrationContext) (any, error) { + if err := ctx.CreateTimer(30 * time.Second).Await(nil); err != nil { + return nil, err + } + if err := ctx.CreateTimer(150 * time.Second).Await(nil); err != nil { + return nil, err + } + return "done", nil + }, + }, + }, + { + name: "sub-orchestrations and send event", + output: `["child-done","child-done"]`, + scenario: sessionScenario{perTurn: 1, orchestrator: func(ctx *OrchestrationContext) (any, error) { + first := ctx.CallSubOrchestrator("child", WithSubOrchestratorInput(1)) + second := ctx.CallSubOrchestrator("child", WithSubOrchestrationInstanceID("fixed-child")) + if err := ctx.SendEvent("other", "hello", "payload"); err != nil { + return nil, err + } + results := make([]string, 2) + if err := first.Await(&results[0]); err != nil { + return nil, err + } + if err := second.Await(&results[1]); err != nil { + return nil, err + } + return results, nil + }}, + }, + { + name: "external events and event channels", + output: `["a","b","c","late"]`, + scenario: sessionScenario{ + extra: map[int][]*protos.HistoryEvent{ + 1: {eventRaised("ch", `"a"`)}, + 2: {eventRaised("ch", `"b"`), eventRaised("ch", `"c"`), eventRaised("single", `"late"`)}, + }, + orchestrator: func(ctx *OrchestrationContext) (any, error) { + channel := NewEventChannel[string](ctx, "ch") + values := []string{} + for range 3 { + values = append(values, channel.Receive(ctx)) + } + var single string + if err := ctx.WaitForSingleEvent("single", -1).Await(&single); err != nil { + return nil, err + } + return append(values, single), nil + }, + }, + }, + { + name: "Go, wait group, WhenAll and select", + output: `"left-0|right-1|all-2,all-3|picked:timer"`, + scenario: sessionScenario{ + perTurn: 1, + extra: map[int][]*protos.HistoryEvent{9: {eventRaised("late", `"x"`)}}, + orchestrator: func(ctx *OrchestrationContext) (any, error) { + results := make([]string, 2) + wg := ctx.NewWaitGroup() + wg.Add(2) + for index, name := range []string{"left", "right"} { + ctx.Go(func(child *OrchestrationContext) { + defer wg.Done() + _ = child.CallActivity(name).Await(&results[index]) + }) + } + wg.Wait(ctx) + all := []Task{ctx.CallActivity("all"), ctx.CallActivity("all")} + if err := ctx.WhenAll(all...); err != nil { + return nil, err + } + allResults := make([]string, 2) + for i, task := range all { + _ = task.Await(&allResults[i]) + } + picked := "" + channel := NewEventChannel[string](ctx, "late") + ctx.Select( + OnTask(ctx.CreateTimer(time.Second), func(Task) { picked = "timer" }), + OnEvent(channel, func(value string) { picked = "event:" + value }), + ) + return strings.Join([]string{results[0], results[1], strings.Join(allResults, ","), "picked:" + picked}, "|"), nil + }, + }, + }, + { + name: "cancellation scopes", + output: `"canceled:after-2"`, + scenario: sessionScenario{perTurn: 1, orchestrator: func(ctx *OrchestrationContext) (any, error) { + timer := ctx.CreateTimer(time.Second) + child, cancel := ctx.WithCancel() + work := child.CallActivity("slow") + winner := ctx.WhenAny(timer, work) + state := "finished" + if winner == timer { + cancel() + if err := work.Await(nil); errors.Is(err, ErrTaskCanceled) { + state = "canceled" + } + } + var after string + if err := ctx.CallActivity("after").Await(&after); err != nil { + return nil, err + } + return state + ":" + after, nil + }}, + }, + { + name: "entities and critical sections", + output: `"get-result|add-result"`, + scenario: sessionScenario{perTurn: 1, orchestrator: func(ctx *OrchestrationContext) (any, error) { + if err := ctx.SignalEntity(other, "reset"); err != nil { + return nil, err + } + var first string + if err := ctx.CallEntity(counter, "get").Await(&first); err != nil { + return nil, err + } + release, err := ctx.LockEntities(counter, other) + if err != nil { + return nil, err + } + var second string + if err := ctx.CallEntity(counter, "add", WithEntityInput(1)).Await(&second); err != nil { + return nil, err + } + release() + return first + "|" + second, nil + }}, + }, + { + name: "canceled lock request after commit", + output: `"lock canceled"`, + check: func(t *testing.T, run sessionRun) { + unlocks := 0 + for _, response := range run.responses { + for _, action := range response.Actions { + if action.GetSendEntityMessage().GetEntityUnlockSent() != nil { + unlocks++ + } + } + } + require.Equal(t, 1, unlocks, "the late grant must be released once") + }, + scenario: sessionScenario{perTurn: 1, orchestrator: func(ctx *OrchestrationContext) (any, error) { + timer := ctx.CreateTimer(time.Second) + child, cancel := ctx.WithCancel() + lockErr := make([]error, 1) + done := ctx.NewWaitGroup() + done.Add(1) + ctx.Go(func(*OrchestrationContext) { + defer done.Done() + _, lockErr[0] = child.LockEntities(counter) + }) + if err := timer.Await(nil); err != nil { + return nil, err + } + cancel() + done.Wait(ctx) + if !errors.Is(lockErr[0], ErrTaskCanceled) { + return nil, fmt.Errorf("unexpected lock result: %w", lockErr[0]) + } + // Wait long enough for the late grant to be released. + if err := ctx.CreateTimer(time.Second).Await(nil); err != nil { + return nil, err + } + return "lock canceled", nil + }}, + }, + { + name: "custom status and NewGuid", + scenario: sessionScenario{orchestrator: func(ctx *OrchestrationContext) (any, error) { + ids := []string{} + for range 3 { + ids = append(ids, ctx.NewGuid()) + ctx.SetCustomStatus(fmt.Sprintf("step-%d", len(ids))) + if err := ctx.CallActivity("work").Await(nil); err != nil { + return nil, err + } + } + ids = append(ids, ctx.NewGuid()) + return ids, nil + }}, + }, + { + name: "suspend with actions created before suspension", + output: `"work-0"`, + check: func(t *testing.T, run sessionRun) { + require.Empty(t, run.responses[1].Actions, "a suspended turn sends no actions") + require.Len(t, run.responses[2].Actions, 1) + require.Equal(t, "work", run.responses[2].Actions[0].GetScheduleTask().GetName()) + }, + scenario: sessionScenario{ + extra: map[int][]*protos.HistoryEvent{ + 1: {eventRaised("go", `""`), suspended()}, + 2: {resumed()}, + }, + orchestrator: func(ctx *OrchestrationContext) (any, error) { + if err := ctx.WaitForSingleEvent("go", -1).Await(nil); err != nil { + return nil, err + } + var result string + err := ctx.CallActivity("work").Await(&result) + return result, err + }, + }, + }, + { + name: "events buffered while suspended", + output: `"payload"`, + scenario: sessionScenario{ + extra: map[int][]*protos.HistoryEvent{ + 1: {suspended(), eventRaised("go", `"payload"`), suspended()}, + 2: {resumed()}, + }, + orchestrator: func(ctx *OrchestrationContext) (any, error) { + var value string + err := ctx.WaitForSingleEvent("go", -1).Await(&value) + return value, err + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + run := runSessionEquivalence(t, tt.scenario) + require.Greater(t, len(run.responses), 2, "scenario should run several turns") + if tt.output != "" { + requireCompletedWith(t, run, tt.output) + } else { + require.True(t, run.sessionEnded) + require.NotNil(t, completionOf(run.responses[len(run.responses)-1])) + } + if tt.check != nil { + tt.check(t, run) + } + }) + } +} + +func TestSessionCustomStatusIsCarriedAcrossTurns(t *testing.T) { + run := runSessionEquivalence(t, sessionScenario{orchestrator: func(ctx *OrchestrationContext) (any, error) { + ctx.SetCustomStatus("waiting") + if err := ctx.CallActivity("a").Await(nil); err != nil { + return nil, err + } + return nil, ctx.CallActivity("b").Await(nil) + }}) + for _, response := range run.responses { + require.Equal(t, "waiting", response.GetCustomStatus().GetValue()) + } +} + +func TestSessionEndsWithOrchestration(t *testing.T) { + waitThen := func(end func(ctx *OrchestrationContext) (any, error)) Orchestrator { + return func(ctx *OrchestrationContext) (any, error) { + if err := ctx.CallActivity("work").Await(nil); err != nil { + return nil, err + } + return end(ctx) + } + } + tests := []struct { + name string + scenario sessionScenario + status protos.OrchestrationStatus + }{ + { + name: "failure", + status: protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, + scenario: sessionScenario{orchestrator: waitThen(func(*OrchestrationContext) (any, error) { + return nil, errors.New("boom") + })}, + }, + { + name: "continue as new", + status: protos.OrchestrationStatus_ORCHESTRATION_STATUS_CONTINUED_AS_NEW, + scenario: sessionScenario{ + extra: map[int][]*protos.HistoryEvent{1: {eventRaised("kept", `"x"`)}}, + orchestrator: waitThen(func(ctx *OrchestrationContext) (any, error) { + ctx.ContinueAsNew("next", WithKeepUnprocessedEvents()) + return nil, nil + }), + }, + }, + { + name: "termination", + status: protos.OrchestrationStatus_ORCHESTRATION_STATUS_TERMINATED, + scenario: sessionScenario{ + extra: map[int][]*protos.HistoryEvent{1: {func() *protos.HistoryEvent { + event := helpers.NewExecutionTerminatedEvent(wrapperspb.String(`"stop"`), false) + event.Timestamp = timestamppb.New(sessionTestStart) + return event + }()}}, + orchestrator: func(ctx *OrchestrationContext) (any, error) { + return nil, ctx.WaitForSingleEvent("never", -1).Await(nil) + }, + }, + }, + { + name: "history limit", + status: protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, + scenario: sessionScenario{ + options: []TaskExecutorOption{WithOrchestrationOptions(OrchestrationOptions{MaxHistoryEvents: 6})}, + orchestrator: func(ctx *OrchestrationContext) (any, error) { + for { + if err := ctx.CallActivity("work").Await(nil); err != nil { + return nil, err + } + } + }, + }, + }, + { + name: "history limit continue as new", + status: protos.OrchestrationStatus_ORCHESTRATION_STATUS_CONTINUED_AS_NEW, + scenario: sessionScenario{ + options: []TaskExecutorOption{WithOrchestrationOptions(OrchestrationOptions{ + MaxHistoryEvents: 6, + OnHistoryLimitExceeded: func(info HistoryLimitInfo) (any, error) { + return info.HistoryLength, nil + }, + })}, + orchestrator: func(ctx *OrchestrationContext) (any, error) { + for { + if err := ctx.CallActivity("work").Await(nil); err != nil { + return nil, err + } + } + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + run := runSessionEquivalence(t, tt.scenario) + require.True(t, run.sessionEnded) + require.Greater(t, len(run.responses), 1) + completed := completionOf(run.responses[len(run.responses)-1]) + require.NotNil(t, completed) + require.Equal(t, tt.status, completed.OrchestrationStatus) + }) + } +} + +func TestSessionEndsOnPartialTurn(t *testing.T) { + run := runSessionEquivalence(t, sessionScenario{ + options: []TaskExecutorOption{WithOrchestrationOptions(OrchestrationOptions{MaxEventsPerTurn: 2})}, + extra: map[int][]*protos.HistoryEvent{ + 1: {eventRaised("e", `1`), eventRaised("e", `2`), eventRaised("e", `3`)}, + }, + orchestrator: func(ctx *OrchestrationContext) (any, error) { + channel := NewEventChannel[int](ctx, "e") + sum := 0 + for range 3 { + sum += channel.Receive(ctx) + } + return sum, nil + }, + }) + require.True(t, run.sessionEnded) + last := run.responses[len(run.responses)-1] + require.NotNil(t, last.NumEventsProcessed) + require.Equal(t, int32(2), last.NumEventsProcessed.GetValue()) + require.Nil(t, completionOf(last)) +} + +func TestSessionEndsOnPanicDuringResume(t *testing.T) { + registry := NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("session", func(ctx *OrchestrationContext) (any, error) { + if err := ctx.CallActivity("work").Await(nil); err != nil { + return nil, err + } + panic("resume panic") + })) + baseline := runtime.NumGoroutine() + executor := NewTaskExecutor(registry).(*taskExecutor) + first := []*protos.HistoryEvent{sessionOrchestratorStarted(sessionTestStart), sessionExecutionStarted("session", nil)} + result, session, err := executor.StartOrchestratorSession(context.Background(), sessionTestInstance, nil, first, nil) + require.NoError(t, err) + require.NotNil(t, session) + session.Commit(result.Response.Actions) + batch := []*protos.HistoryEvent{ + sessionOrchestratorStarted(sessionTestStart.Add(time.Minute)), + defaultResponse(result.Response.Actions[0]), + } + result, next, err := session.Resume(context.Background(), batch, len(first)+1) + require.NoError(t, err) + require.Nil(t, next) + completed := completionOf(result.Response) + require.NotNil(t, completed) + require.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, completed.OrchestrationStatus) + require.Contains(t, completed.GetFailureDetails().GetErrorMessage(), "resume panic") + requireNoGoroutineLeak(t, baseline) +} + +// TestStartOrchestratorSessionMatchesExecuteOrchestrator checks first turns +// that cannot keep a session. They must return the same result or error as +// ExecuteOrchestrator and no session. +func TestStartOrchestratorSessionMatchesExecuteOrchestrator(t *testing.T) { + started := func(name string, version string) *protos.HistoryEvent { + event := sessionExecutionStarted(name, nil) + if version != "" { + event.GetExecutionStarted().Version = wrapperspb.String(version) + } + return event + } + tests := []struct { + name string + options []TaskExecutorOption + old []*protos.HistoryEvent + new []*protos.HistoryEvent + params *protos.OrchestratorEntityParameters + wantError bool + }{ + {name: "completes", new: []*protos.HistoryEvent{sessionOrchestratorStarted(sessionTestStart), started("done", "")}}, + {name: "fails", new: []*protos.HistoryEvent{sessionOrchestratorStarted(sessionTestStart), started("fail", "")}}, + {name: "panics", new: []*protos.HistoryEvent{sessionOrchestratorStarted(sessionTestStart), started("panic", "")}}, + {name: "continues as new", new: []*protos.HistoryEvent{sessionOrchestratorStarted(sessionTestStart), started("again", "")}}, + {name: "not registered", new: []*protos.HistoryEvent{sessionOrchestratorStarted(sessionTestStart), started("missing", "")}}, + { + name: "not registered reject", + options: []TaskExecutorOption{ + WithOrchestratorNotFoundStrategy(OrchestratorNotFoundReject), + }, + new: []*protos.HistoryEvent{sessionOrchestratorStarted(sessionTestStart), started("missing", "")}, + wantError: true, + }, + { + name: "version fail", + options: []TaskExecutorOption{WithVersioning(VersioningOptions{Version: "2.0", MatchStrategy: VersionMatchStrict, FailureStrategy: VersionFailureFail})}, + new: []*protos.HistoryEvent{sessionOrchestratorStarted(sessionTestStart), started("wait", "1.0")}, + }, + { + name: "version reject", + options: []TaskExecutorOption{WithVersioning(VersioningOptions{Version: "2.0", MatchStrategy: VersionMatchStrict, FailureStrategy: VersionFailureReject})}, + new: []*protos.HistoryEvent{sessionOrchestratorStarted(sessionTestStart), started("wait", "1.0")}, + wantError: true, + }, + { + name: "unsupported entity parameters", + new: []*protos.HistoryEvent{sessionOrchestratorStarted(sessionTestStart), started("wait", "")}, + params: &protos.OrchestratorEntityParameters{}, + wantError: true, + }, + { + name: "non-deterministic replay", + old: []*protos.HistoryEvent{ + sessionOrchestratorStarted(sessionTestStart), + started("wait", ""), + helpers.NewTimerCreatedEvent(0, timestamppb.New(sessionTestStart)), + }, + new: []*protos.HistoryEvent{sessionOrchestratorStarted(sessionTestStart.Add(time.Minute))}, + }, + { + name: "partial turn", + options: []TaskExecutorOption{ + WithOrchestrationOptions(OrchestrationOptions{MaxEventsPerTurn: 1}), + }, + new: []*protos.HistoryEvent{ + sessionOrchestratorStarted(sessionTestStart), + started("wait", ""), + eventRaised("x", `1`), + }, + }, + { + name: "rewind", + old: []*protos.HistoryEvent{ + sessionOrchestratorStarted(sessionTestStart), + started("fail", ""), + { + EventId: -1, + Timestamp: timestamppb.New(sessionTestStart), + EventType: &protos.HistoryEvent_ExecutionCompleted{ExecutionCompleted: &protos.ExecutionCompletedEvent{ + OrchestrationStatus: protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, + FailureDetails: &protos.TaskFailureDetails{ErrorType: "x", ErrorMessage: "y"}, + }}, + }, + }, + new: []*protos.HistoryEvent{ + sessionOrchestratorStarted(sessionTestStart.Add(time.Minute)), + { + EventId: -1, + Timestamp: timestamppb.New(sessionTestStart), + EventType: &protos.HistoryEvent_ExecutionRewound{ExecutionRewound: &protos.ExecutionRewoundEvent{}}, + }, + }, + }, + } + registry := NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("done", func(*OrchestrationContext) (any, error) { return "ok", nil })) + require.NoError(t, registry.AddOrchestratorN("fail", func(*OrchestrationContext) (any, error) { return nil, errors.New("failed") })) + require.NoError(t, registry.AddOrchestratorN("panic", func(*OrchestrationContext) (any, error) { panic("first turn") })) + require.NoError(t, registry.AddOrchestratorN("again", func(ctx *OrchestrationContext) (any, error) { + ctx.ContinueAsNew(1) + return nil, nil + })) + require.NoError(t, registry.AddOrchestratorN("wait", func(ctx *OrchestrationContext) (any, error) { + return nil, ctx.WaitForSingleEvent("never", -1).Await(nil) + })) + require.NoError(t, registry.AddOrchestratorNVersion("wait", "1.0", func(ctx *OrchestrationContext) (any, error) { + return nil, ctx.WaitForSingleEvent("never", -1).Await(nil) + })) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + params := tt.params + if params == nil { + params = supportedEntityParameters() + } + baseline := runtime.NumGoroutine() + executor := NewTaskExecutor(registry, tt.options...).(*taskExecutor) + want, wantErr := executor.ExecuteOrchestrator(context.Background(), sessionTestInstance, tt.old, tt.new, params) + got, session, gotErr := executor.StartOrchestratorSession(context.Background(), sessionTestInstance, tt.old, tt.new, params) + require.Nil(t, session) + if tt.wantError { + require.Error(t, wantErr) + require.EqualError(t, gotErr, wantErr.Error()) + require.Nil(t, got) + } else { + require.NoError(t, wantErr) + require.NoError(t, gotErr) + switch tt.name { + case "panics": + // Stack traces differ between the two calls. + require.Equal(t, completionOf(want.Response).OrchestrationStatus, completionOf(got.Response).OrchestrationStatus) + case "rewind": + // Rewind generates a new execution ID on every call. + require.Len(t, got.Response.Actions, len(want.Response.Actions)) + default: + requireSameResponse(t, 0, want.Response, got.Response) + } + } + requireNoGoroutineLeak(t, baseline) + }) + } +} + +func startWaitingSession(t *testing.T, orchestrator Orchestrator) (*OrchestrationContext, OrchestratorSession, *protos.OrchestratorResponse) { + t.Helper() + registry := NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("session", orchestrator)) + executor := NewTaskExecutor(registry).(*taskExecutor) + result, session, err := executor.StartOrchestratorSession( + context.Background(), + sessionTestInstance, + nil, + []*protos.HistoryEvent{sessionOrchestratorStarted(sessionTestStart), sessionExecutionStarted("session", nil)}, + nil, + ) + require.NoError(t, err) + require.NotNil(t, session) + return session.(*orchestratorSession).ctx, session, result.Response +} + +func TestSessionStateMachine(t *testing.T) { + waiting := func(ctx *OrchestrationContext) (any, error) { + return nil, ctx.CallActivity("work").Await(nil) + } + validBatch := []*protos.HistoryEvent{sessionOrchestratorStarted(sessionTestStart.Add(time.Minute))} + + t.Run("resume before commit", func(t *testing.T) { + _, session, _ := startWaitingSession(t, waiting) + _, next, err := session.Resume(context.Background(), validBatch, 3) + require.ErrorContains(t, err, "before the last turn was committed") + require.Nil(t, next) + _, _, err = session.Resume(context.Background(), validBatch, 3) + require.Error(t, err) + }) + t.Run("commit twice", func(t *testing.T) { + _, session, response := startWaitingSession(t, waiting) + session.Commit(response.Actions) + session.Commit(response.Actions) + _, _, err := session.Resume(context.Background(), validBatch, 3) + require.ErrorContains(t, err, "commit is out of order") + }) + t.Run("resume after close", func(t *testing.T) { + _, session, response := startWaitingSession(t, waiting) + session.Commit(response.Actions) + session.Close() + session.Close() + _, _, err := session.Resume(context.Background(), validBatch, 3) + require.ErrorIs(t, err, errOrchestratorSessionClosed) + }) + invalid := []struct { + name string + batch []*protos.HistoryEvent + base int + message string + }{ + {name: "empty", batch: nil, base: 3, message: "must start with an OrchestratorStarted"}, + {name: "no orchestrator started", batch: []*protos.HistoryEvent{eventRaised("x", `1`)}, base: 3, message: "must start with an OrchestratorStarted"}, + {name: "negative base", batch: validBatch, base: -1, message: "must not be negative"}, + {name: "new execution", batch: []*protos.HistoryEvent{validBatch[0], sessionExecutionStarted("session", nil)}, base: 3, message: "must not start a new execution"}, + {name: "rewind", batch: []*protos.HistoryEvent{validBatch[0], {EventType: &protos.HistoryEvent_ExecutionRewound{ExecutionRewound: &protos.ExecutionRewoundEvent{}}}}, base: 3, message: "rewind"}, + {name: "nil event", batch: []*protos.HistoryEvent{validBatch[0], nil}, base: 3, message: "nil events"}, + } + for _, tt := range invalid { + t.Run("invalid batch "+tt.name, func(t *testing.T) { + ctx, session, response := startWaitingSession(t, waiting) + session.Commit(response.Actions) + _, next, err := session.Resume(context.Background(), tt.batch, tt.base) + require.ErrorContains(t, err, tt.message) + require.Nil(t, next) + require.Nil(t, ctx.scheduler, "an invalid batch must unload the session") + _, _, err = session.Resume(context.Background(), validBatch, 3) + require.Error(t, err) + }) + } +} + +func TestSessionCommitRetiresOnlySentActions(t *testing.T) { + ctx, session, response := startWaitingSession(t, func(ctx *OrchestrationContext) (any, error) { + return nil, ctx.WhenAll(ctx.CallActivity("a"), ctx.CallActivity("b")) + }) + require.Len(t, response.Actions, 2) + session.Commit(response.Actions[:1]) + require.Len(t, ctx.pendingActions, 1) + result, next, err := session.Resume(context.Background(), []*protos.HistoryEvent{ + sessionOrchestratorStarted(sessionTestStart.Add(time.Minute)), + }, 4) + require.NoError(t, err) + require.NotNil(t, next) + require.Len(t, result.Response.Actions, 1) + require.True(t, proto.Equal(response.Actions[1], result.Response.Actions[0]), "an unsent action must be sent again") + next.Close() +} + +func TestSessionHistoryLengthUsesHistoryBase(t *testing.T) { + var lengths []int + registry := NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("session", func(ctx *OrchestrationContext) (any, error) { + for { + if err := ctx.WaitForSingleEvent("e", -1).Await(nil); err != nil { + return nil, err + } + } + })) + executor := newSessionTestExecutor(registry, &lengths, + WithOrchestrationOptions(OrchestrationOptions{MaxHistoryEvents: 100})) + result, session, err := executor.StartOrchestratorSession(context.Background(), sessionTestInstance, nil, + []*protos.HistoryEvent{sessionOrchestratorStarted(sessionTestStart), sessionExecutionStarted("session", nil)}, nil) + require.NoError(t, err) + require.Equal(t, []int{2}, lengths) + session.Commit(result.Response.Actions) + + batch := []*protos.HistoryEvent{sessionOrchestratorStarted(sessionTestStart.Add(time.Minute)), eventRaised("e", `1`)} + result, session, err = session.Resume(context.Background(), batch, 98) + require.NoError(t, err) + require.NotNil(t, session) + require.Equal(t, []int{2, 100}, lengths) + session.Commit(result.Response.Actions) + + result, session, err = session.Resume(context.Background(), batch, 99) + require.NoError(t, err) + require.Nil(t, session, "the history limit must end the session") + completed := completionOf(result.Response) + require.NotNil(t, completed) + require.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, completed.OrchestrationStatus) + require.Equal(t, []int{2, 100, 101}, lengths) +} + +func TestSessionCloseEmitsNoDurableWork(t *testing.T) { + baseline := runtime.NumGoroutine() + deferredBlocked := 0 + ctx, session, response := startWaitingSession(t, func(ctx *OrchestrationContext) (any, error) { + for range 3 { + ctx.Go(func(child *OrchestrationContext) { + defer func() { + if recovered := recover(); recovered != nil { + if errors.Is(recovered.(error), ErrTaskBlocked) { + deferredBlocked++ + } + panic(recovered) + } + }() + defer func() { + _ = child.CallActivity("cleanup") + }() + _ = child.CreateTimer(time.Hour).Await(nil) + }) + } + defer func() { + ctx.SetCustomStatus("changed") + _ = ctx.SignalEntity(api.NewEntityID("counter", "x"), "op") + }() + return nil, ctx.WaitForSingleEvent("never", -1).Await(nil) + }) + require.Len(t, response.Actions, 3) + session.Commit(response.Actions) + sequence := ctx.sequenceNumber + status := ctx.customStatus + require.Greater(t, runtime.NumGoroutine(), baseline) + + session.Close() + require.Equal(t, sequence, ctx.sequenceNumber, "Close must not schedule durable work") + require.Empty(t, ctx.pendingActions) + require.Equal(t, status, ctx.customStatus) + require.Equal(t, 3, deferredBlocked) + requireNoGoroutineLeak(t, baseline) +} From ce4b7afaa8623037f2c2b53c6bdb16bfae693ada Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Fri, 6 Nov 2026 16:22:01 -0800 Subject: [PATCH 02/21] Drop processed history while an orchestration session waits A continuing session kept the full first-turn history and every processed batch. Clear them after the turn result is built. The coroutines and the event buffers keep what later turns need. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- task/orchestrator.go | 11 +++++++++++ task/session.go | 1 + task/session_test.go | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+) diff --git a/task/orchestrator.go b/task/orchestrator.go index b0f033c7..0fbc7294 100644 --- a/task/orchestrator.go +++ b/task/orchestrator.go @@ -425,6 +425,17 @@ func (ctx *OrchestrationContext) commit(sent []*protos.OrchestratorAction) { } } +// releaseProcessedHistory drops the events of a finished turn while the +// orchestration waits for its next batch. The live coroutines and the event +// buffers keep everything that later turns need, so a long session does not +// keep its full history in memory. +func (ctx *OrchestrationContext) releaseProcessedHistory() { + ctx.historyBase = ctx.HistoryLength() + ctx.oldEvents = nil + ctx.newEvents = nil + ctx.historyIndex = 0 +} + // resume runs one more turn on the loaded orchestration. The batch contains // only new events, and historyBase is the number of events that the backend // has already stored for this execution. diff --git a/task/session.go b/task/session.go index da3dee65..a2dd6762 100644 --- a/task/session.go +++ b/task/session.go @@ -86,6 +86,7 @@ func (s *orchestratorSession) finishTurn( s.Close() return results, nil, nil } + s.ctx.releaseProcessedHistory() s.state = sessionAwaitingCommit return results, s, nil } diff --git a/task/session_test.go b/task/session_test.go index 70cdd927..9402d3ff 100644 --- a/task/session_test.go +++ b/task/session_test.go @@ -1029,6 +1029,43 @@ func TestSessionHistoryLengthUsesHistoryBase(t *testing.T) { require.Equal(t, []int{2, 100, 101}, lengths) } +func TestSessionDropsProcessedHistoryBetweenTurns(t *testing.T) { + ctx, session, response := startWaitingSession(t, func(ctx *OrchestrationContext) (any, error) { + var value string + if err := ctx.WaitForSingleEvent("go", -1).Await(&value); err != nil { + return nil, err + } + return value, ctx.CallActivity("work").Await(nil) + }) + require.Nil(t, ctx.oldEvents) + require.Nil(t, ctx.newEvents) + require.Equal(t, 2, ctx.historyBase) + session.Commit(response.Actions) + + // The event is buffered while suspended, and it must survive the cleanup. + batch := []*protos.HistoryEvent{ + sessionOrchestratorStarted(sessionTestStart.Add(time.Minute)), + suspended(), + eventRaised("go", `"payload"`), + } + result, next, err := session.Resume(context.Background(), batch, 2) + require.NoError(t, err) + require.NotNil(t, next) + require.Empty(t, result.Response.Actions) + require.Nil(t, ctx.newEvents) + require.Len(t, ctx.suspendedEvents, 1) + next.Commit(result.Response.Actions) + + result, next, err = next.Resume(context.Background(), + []*protos.HistoryEvent{sessionOrchestratorStarted(sessionTestStart.Add(2 * time.Minute)), resumed()}, 5) + require.NoError(t, err) + require.NotNil(t, next) + require.Len(t, result.Response.Actions, 1) + require.Equal(t, "work", result.Response.Actions[0].GetScheduleTask().GetName()) + require.Nil(t, ctx.newEvents) + next.Close() +} + func TestSessionCloseEmitsNoDurableWork(t *testing.T) { baseline := runtime.NumGoroutine() deferredBlocked := 0 From 7e90ee29dfd98ec6152b5a07eca6e1482dcfc4cf Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Fri, 6 Nov 2026 16:31:15 -0800 Subject: [PATCH 03/21] Route extended-session streams to completion connections Session completion streams now use the least-loaded completion connection with free budget: 50 streams, half of the DTS per-connection limit. When no connection has budget, NewStream fails before sending anything, so the worker can use the unary RPC. The release RPCs are completion methods. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- client/grpc_worker_connection.go | 85 ++++++++++++++++++++++++++++++-- 1 file changed, 82 insertions(+), 3 deletions(-) diff --git a/client/grpc_worker_connection.go b/client/grpc_worker_connection.go index acde82a8..b8d275c7 100644 --- a/client/grpc_worker_connection.go +++ b/client/grpc_worker_connection.go @@ -18,6 +18,16 @@ import ( const maxWorkerCompletionConnections = 8 +// workerSessionStreamsPerConnection is the extended-session stream budget of +// one completion connection: half of the 100 HTTP/2 streams per connection +// that DTS documents, so other completions always have room. +const workerSessionStreamsPerConnection = 50 + +// errSessionStreamBudgetFull means that no completion connection can open +// another extended-session stream. Nothing was sent, so the caller completes +// the work item with the unary RPC instead. +var errSessionStreamBudgetFull = errors.New("worker extended-session stream budget is full") + func validateWorkerCompletionBudget(n int) error { if n < 1 || n > maxWorkerCompletionConnections { return fmt.Errorf("worker completion connections must be between 1 and %d", maxWorkerCompletionConnections) @@ -67,7 +77,10 @@ type workerCompletionTransport struct { closers []io.Closer next atomic.Uint64 closeOnce sync.Once - closeErr error + + sessionMu sync.Mutex + sessionStreams []int + closeErr error } func newWorkerCompletionTransport( @@ -158,7 +171,9 @@ func isWorkerCompletionMethod(method string) bool { protos.TaskHubSidecarService_CompleteEntityTask_FullMethodName, protos.TaskHubSidecarService_AbandonTaskActivityWorkItem_FullMethodName, protos.TaskHubSidecarService_AbandonTaskOrchestratorWorkItem_FullMethodName, - protos.TaskHubSidecarService_AbandonTaskEntityWorkItem_FullMethodName: + protos.TaskHubSidecarService_AbandonTaskEntityWorkItem_FullMethodName, + protos.TaskHubSidecarService_ReleaseTaskOrchestratorWorkItem_FullMethodName, + protos.TaskHubSidecarService_ReleaseTaskEntityWorkItem_FullMethodName: return true default: return false @@ -171,7 +186,71 @@ func (c *workerCompletionTransport) NewStream( method string, opts ...grpc.CallOption, ) (grpc.ClientStream, error) { - return c.intake.NewStream(ctx, desc, method, opts...) + if !isWorkerSessionStreamMethod(method) { + return c.intake.NewStream(ctx, desc, method, opts...) + } + index, ok := c.reserveSessionStream() + if !ok { + return nil, errSessionStreamBudgetFull + } + stream, err := c.completions[index].NewStream(ctx, desc, method, opts...) + if err != nil { + c.releaseSessionStream(index) + return nil, err + } + budgeted := &budgetedClientStream{ClientStream: stream} + var once sync.Once + budgeted.release = func() { once.Do(func() { c.releaseSessionStream(index) }) } + // Callers cancel the stream context when they finish with the stream. + context.AfterFunc(ctx, budgeted.release) + return budgeted, nil +} + +func isWorkerSessionStreamMethod(method string) bool { + return method == protos.TaskHubSidecarService_CompleteOrchestratorTaskWithExtendedSession_FullMethodName || + method == protos.TaskHubSidecarService_CompleteEntityTaskWithExtendedSession_FullMethodName +} + +// reserveSessionStream picks the least-loaded completion connection that +// still has session stream budget. +func (c *workerCompletionTransport) reserveSessionStream() (int, bool) { + c.sessionMu.Lock() + defer c.sessionMu.Unlock() + if c.sessionStreams == nil { + c.sessionStreams = make([]int, len(c.completions)) + } + best := -1 + for i, streams := range c.sessionStreams { + if streams < workerSessionStreamsPerConnection && (best < 0 || streams < c.sessionStreams[best]) { + best = i + } + } + if best < 0 { + return 0, false + } + c.sessionStreams[best]++ + return best, true +} + +func (c *workerCompletionTransport) releaseSessionStream(index int) { + c.sessionMu.Lock() + c.sessionStreams[index]-- + c.sessionMu.Unlock() +} + +// budgetedClientStream returns its session stream budget when the stream +// ends with any status or when its context ends. +type budgetedClientStream struct { + grpc.ClientStream + release func() +} + +func (s *budgetedClientStream) RecvMsg(m any) error { + err := s.ClientStream.RecvMsg(m) + if err != nil { + s.release() + } + return err } func (c *workerCompletionTransport) Close() error { From 5671231aca1b3216c59f921f6d2caa91d46c3aa9 Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Fri, 6 Nov 2026 16:31:15 -0800 Subject: [PATCH 04/21] Add extended sessions to the gRPC worker WithExtendedSessions keeps orchestrations and entities loaded between turns. One goroutine owns each session turn: it sends the streaming completion, waits for the acknowledgement and the next batch with its own timers, and is the only caller of release and abandon. - The turn count is fixed when the turn starts. Retries and the unary fallback reuse it, and it advances only on the acknowledgement. The count and token are set right before each send. - Recovery follows the plan: transient errors before the acknowledgement retry, other errors abandon, failures after it release, and Unimplemented turns sessions off until the next reconnect. - Release and abandon stop at one absolute deadline. After it, the worker ends the intake stream so the service abandons the generation's work. - Sessions close without release when their intake generation ends, and waiting sessions are released on shutdown. - Entities run each batch with the state of the last accepted result. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- client/grpc_worker.go | 29 +- client/grpc_worker_processor.go | 391 ++++++++++-- client/grpc_worker_session.go | 489 +++++++++++++++ client/grpc_worker_session_test.go | 961 +++++++++++++++++++++++++++++ 4 files changed, 1818 insertions(+), 52 deletions(-) create mode 100644 client/grpc_worker_session.go create mode 100644 client/grpc_worker_session_test.go diff --git a/client/grpc_worker.go b/client/grpc_worker.go index ea367103..8d4ab5c0 100644 --- a/client/grpc_worker.go +++ b/client/grpc_worker.go @@ -12,6 +12,7 @@ import ( "slices" "strings" "sync" + "sync/atomic" "time" "github.com/microsoft/durabletask-go/api" @@ -118,6 +119,10 @@ type taskHubGrpcWorkerOptions struct { unversionedOrchestrators map[string]struct{} unversionedActivities map[string]struct{} reconnectRandom randomInt64N + extendedSessions *ExtendedSessionOptions + sessionCleanupTimeout time.Duration + // sessionTimerFn replaces the session timers in tests. + sessionTimerFn func(sessionTimerKind, time.Duration) <-chan time.Time // waitFn overrides every delay the worker imposes on itself: reconnect // backoff, transient RPC retry backoff, and work-item abandon delays. It is // only set by tests so the deterministic delay schedule can be observed @@ -144,6 +149,7 @@ func defaultTaskHubGrpcWorkerOptions() taskHubGrpcWorkerOptions { maxStreamedHistoryBytes: DefaultMaxStreamedHistoryBytes, maxOrchestratorCompletionBytes: DefaultMaxOrchestratorCompletionBytes, reconnectRandom: rand.Int64N, + sessionCleanupTimeout: defaultSessionCleanupTimeout, capabilities: []WorkerCapability{WorkerCapabilityHistoryStreaming}, } } @@ -537,6 +543,7 @@ type grpcWorkerRun struct { orchestrationSlots chan struct{} activitySlots chan struct{} entitySlots chan struct{} + sessions *sessionManager pending sync.WaitGroup retired sync.WaitGroup err error @@ -547,6 +554,11 @@ type grpcWorkerConnection struct { stream workItemsStream cancelStream context.CancelFunc closer io.Closer + // streamCtx ends with this intake generation. Sessions dispatched by the + // generation end with it. + streamCtx context.Context + sessionsUnsupported atomic.Bool + sessionCleanupFailed atomic.Bool pendingMu sync.Mutex pendingChanged *sync.Cond @@ -590,6 +602,14 @@ func (c *grpcWorkerConnection) waitForWorkItems() { c.waitForWorkItemsLocked() } +// abortForSessionCleanup ends the intake stream after a session work item +// could not be released or abandoned. The service then abandons every work +// item of this generation, and the worker reconnects. +func (c *grpcWorkerConnection) abortForSessionCleanup() { + c.sessionCleanupFailed.Store(true) + c.cancelStream() +} + func (c *grpcWorkerConnection) stopIntake() { c.pendingMu.Lock() defer c.pendingMu.Unlock() @@ -707,6 +727,9 @@ func newTaskHubGrpcWorker( return nil, err } } + if err := options.resolveExtendedSessionLimits(); err != nil { + return nil, err + } hasLargePayloadCapability := slices.Contains(options.capabilities, WorkerCapabilityLargePayloads) if hasLargePayloadCapability && options.largePayloads == nil { return nil, fmt.Errorf("large-payload capability requires worker large-payload options") @@ -1067,6 +1090,9 @@ func (w *TaskHubGrpcWorker) begin(ctx context.Context) (*grpcWorkerRun, *grpcWor activitySlots: make(chan struct{}, w.options.maxConcurrentActivities), entitySlots: make(chan struct{}, w.options.maxConcurrentEntities), } + if w.options.extendedSessions != nil { + run.sessions = newSessionManager(w.options.extendedSessions) + } w.run = run w.lastErr = nil w.mu.Unlock() @@ -1236,6 +1262,7 @@ func (w *TaskHubGrpcWorker) connect(ctx context.Context) (*grpcWorkerConnection, stream: stream, cancelStream: cancelStream, closer: closer, + streamCtx: streamCtx, }, nil } @@ -1415,7 +1442,7 @@ func isTransientWorkerError(err error) bool { if errors.Is(err, ErrStreamedHistoryLimitExceeded) { return false } - if errors.Is(err, io.EOF) || errors.Is(err, errSilentDisconnect) { + if errors.Is(err, io.EOF) || errors.Is(err, errSilentDisconnect) || errors.Is(err, errSessionCleanupFailed) { return true } grpcStatus, ok := status.FromError(err) diff --git a/client/grpc_worker_processor.go b/client/grpc_worker_processor.go index 92675571..8797781e 100644 --- a/client/grpc_worker_processor.go +++ b/client/grpc_worker_processor.go @@ -17,6 +17,7 @@ import ( "github.com/microsoft/durabletask-go/internal/largepayload" "github.com/microsoft/durabletask-go/internal/protos" "github.com/microsoft/durabletask-go/task" + "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" @@ -73,7 +74,8 @@ func (w *TaskHubGrpcWorker) consumeConnection(run *grpcWorkerRun, connection *gr } case *protos.WorkItem_EntityRequestV2: if err := w.dispatchEntity(run, connection, workItem.GetCompletionToken(), func(ctx context.Context) { - w.processEntityV2(ctx, connection.client, workItem.GetCompletionToken(), request.EntityRequestV2) + scope := &workerSessionScope{run: run, connection: connection} + w.processEntityWorkItem(ctx, connection.client, scope, workItem.GetCompletionToken(), request.EntityRequestV2) }); err != nil { return observedMessage, err } @@ -94,6 +96,9 @@ func (w *TaskHubGrpcWorker) receiveWorkItem(run *grpcWorkerRun, connection *grpc } workItem, err := recvBeforeSilenceTimeout(connection.stream.Recv, cancelOnSilence, w.options.silentDisconnectTimeout) if err != nil { + if connection.sessionCleanupFailed.Load() { + return nil, errSessionCleanupFailed + } return nil, err } if workItem == nil { @@ -156,7 +161,9 @@ func (w *TaskHubGrpcWorker) dispatchOrchestration( ) error { return w.dispatch(run, connection, run.orchestrationSlots, func(ctx context.Context) { w.abandonOrchestration(ctx, connection.client, completionToken) }, - func(ctx context.Context) { w.processOrchestration(ctx, connection.client, completionToken, request) }, + func(ctx context.Context) { + w.processOrchestrationWorkItem(ctx, connection.client, &workerSessionScope{run: run, connection: connection}, completionToken, request) + }, ) } @@ -227,6 +234,18 @@ func (w *TaskHubGrpcWorker) processOrchestration( client protos.TaskHubSidecarServiceClient, completionToken string, request *protos.OrchestratorRequest, +) { + w.processOrchestrationWorkItem(ctx, client, nil, completionToken, request) +} + +// processOrchestrationWorkItem runs an orchestration work item. When scope +// allows extended sessions, the orchestration can stay loaded for later turns. +func (w *TaskHubGrpcWorker) processOrchestrationWorkItem( + ctx context.Context, + client protos.TaskHubSidecarServiceClient, + scope *workerSessionScope, + completionToken string, + request *protos.OrchestratorRequest, ) { pastEvents := request.PastEvents if request.RequiresHistoryStreaming { @@ -260,13 +279,29 @@ func (w *TaskHubGrpcWorker) processOrchestration( } } - results, err := w.executor.ExecuteOrchestrator( - ctx, - api.InstanceID(request.InstanceId), - pastEvents, - request.NewEvents, - request.EntityParameters, + var ( + results *task.ExecutionResults + session task.OrchestratorSession + err error ) + sessionExecutor, canSession := w.executor.(task.SessionExecutor) + if canSession && scope.available() { + results, session, err = sessionExecutor.StartOrchestratorSession( + ctx, + api.InstanceID(request.InstanceId), + pastEvents, + request.NewEvents, + request.EntityParameters, + ) + } else { + results, err = w.executor.ExecuteOrchestrator( + ctx, + api.InstanceID(request.InstanceId), + pastEvents, + request.NewEvents, + request.EntityParameters, + ) + } var delayed workItemAbandonDelayError if errors.As(err, &delayed) { w.logger.Warnf("%s: orchestration work item rejected; abandoning it: %v", request.InstanceId, err) @@ -280,14 +315,52 @@ func (w *TaskHubGrpcWorker) processOrchestration( return } + turn := orchestrationTurn{ + instanceID: request.InstanceId, + token: completionToken, + traceContext: request.OrchestrationTraceContext, + parentTrace: executionStartedTraceContext(pastEvents, request.NewEvents), + } + response, ok := w.orchestratorResponse(ctx, client, turn, results, err) + if !ok { + if session != nil { + session.Close() + } + return + } + if session != nil { + w.runOrchestrationSession(ctx, client, scope, turn, session, response, historySize(pastEvents, request.NewEvents)) + return + } + w.completeOrchestration(ctx, client, turn, response, 0) +} + +// orchestrationTurn holds what every turn of one orchestration work item +// needs to build and send its response. +type orchestrationTurn struct { + instanceID string + token string + traceContext *protos.OrchestrationTraceContext + parentTrace *protos.TraceContext +} + +// orchestratorResponse builds the response for one turn. It returns false +// after it abandoned the work item. +func (w *TaskHubGrpcWorker) orchestratorResponse( + ctx context.Context, + client protos.TaskHubSidecarServiceClient, + turn orchestrationTurn, + results *task.ExecutionResults, + err error, +) (*protos.OrchestratorResponse, bool) { response := &protos.OrchestratorResponse{ - InstanceId: request.InstanceId, - CompletionToken: completionToken, - OrchestrationTraceContext: request.OrchestrationTraceContext, + InstanceId: turn.instanceID, + CompletionToken: turn.token, + OrchestrationTraceContext: turn.traceContext, } switch { case err != nil: - w.logger.Errorf("%s: orchestration execution failed: %v", request.InstanceId, err) + w.logger.Errorf("%s: orchestration execution failed: %v", turn.instanceID, err) response.Actions = []*protos.OrchestratorAction{helpers.NewCompleteOrchestrationAction( -1, protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, @@ -308,38 +381,149 @@ func (w *TaskHubGrpcWorker) processOrchestration( )} default: response = results.Response - response.InstanceId = request.InstanceId - response.CompletionToken = completionToken + response.InstanceId = turn.instanceID + response.CompletionToken = turn.token if response.OrchestrationTraceContext == nil { - response.OrchestrationTraceContext = request.OrchestrationTraceContext + response.OrchestrationTraceContext = turn.traceContext } - if err := populateOrchestratorActionTraceContexts( - response.Actions, - pastEvents, - request.NewEvents, - ); err != nil { - w.logger.Warnf("%s: failed to populate action trace contexts: %v", request.InstanceId, err) + if err := populateActionTraceContexts(response.Actions, turn.parentTrace); err != nil { + w.logger.Warnf("%s: failed to populate action trace contexts: %v", turn.instanceID, err) } } if err := largepayload.TransformOrchestratorResponse(ctx, w.options.largePayloads, response); err != nil { - w.logger.Errorf("%s: failed to externalize orchestration response payloads: %v", request.InstanceId, err) - w.abandonOrchestration(ctx, client, completionToken) - return + w.logger.Errorf("%s: failed to externalize orchestration response payloads: %v", turn.instanceID, err) + w.abandonOrchestration(ctx, client, turn.token) + return nil, false } + return failOversizedOrchestratorResponse(response, w.options.maxOrchestratorCompletionBytes), true +} - response = failOversizedOrchestratorResponse(response, w.options.maxOrchestratorCompletionBytes) - err = w.executeRPCWithRetry(ctx, "complete orchestration task", func(callCtx context.Context) error { +// completeOrchestration sends a turn with the unary RPC. The count and token +// are set right before sending, because earlier steps may rebuild the +// response. +func (w *TaskHubGrpcWorker) completeOrchestration( + ctx context.Context, + client protos.TaskHubSidecarServiceClient, + turn orchestrationTurn, + response *protos.OrchestratorResponse, + completionCount int32, +) { + response.CompletionToken = turn.token + response.CompletionCount = completionCount + err := w.executeRPCWithRetry(ctx, "complete orchestration task", func(callCtx context.Context) error { _, callErr := client.CompleteOrchestratorTask(callCtx, response) return callErr }) if err != nil { if isWorkItemGone(err) { - w.logger.Warnf("%s: orchestration work item was no longer available before completion", request.InstanceId) + w.logger.Warnf("%s: orchestration work item was no longer available before completion", turn.instanceID) return } - w.logger.Errorf("%s: failed to complete orchestration work item: %v", request.InstanceId, err) - w.abandonOrchestration(ctx, client, completionToken) + w.logger.Errorf("%s: failed to complete orchestration work item: %v", turn.instanceID, err) + w.abandonOrchestration(ctx, client, turn.token) + } +} + +// runOrchestrationSession keeps an orchestration loaded while the service +// sends new batches. This goroutine owns the session: it sends each turn, +// waits for the next batch, and is the only caller of release and abandon. +func (w *TaskHubGrpcWorker) runOrchestrationSession( + ctx context.Context, + client protos.TaskHubSidecarServiceClient, + scope *workerSessionScope, + turn orchestrationTurn, + session task.OrchestratorSession, + response *protos.OrchestratorResponse, + sessionBytes int64, +) { + lease := scope.run.sessions.acquire(orchestrationSessionKind) + defer lease.release() + var turnCount int32 + for { + if session != nil && (lease == nil || !scope.available() || !canContinueOrchestrationSession(response)) { + session.Close() + session = nil + } + if session == nil { + w.completeOrchestration(ctx, client, turn, response, turnCount) + return + } + var historyBase int32 + outcome, batch := w.awaitSessionBatch(ctx, scope, lease, sessionCall{ + instanceID: turn.instanceID, + open: func(callCtx context.Context) (grpc.ServerStreamingClient[protos.CompleteTaskWithExtendedSessionResponse], error) { + response.CompletionToken = turn.token + response.CompletionCount = turnCount + return client.CompleteOrchestratorTaskWithExtendedSession(callCtx, response) + }, + onAck: func(ack *protos.ExtendedSessionAck) int64 { + session.Commit(response.Actions) + historyBase = ack.GetHistoryEventCount() + turnCount++ + return sessionBytes + }, + release: releaseOrchestrationOnce(client, turn.token), + abandon: abandonOrchestrationOnce(client, turn.token), + }) + switch outcome { + case sessionUseUnary: + session.Close() + session = nil + continue + case sessionEnded: + session.Close() + return + } + + payloadRequest := &protos.OrchestratorRequest{NewEvents: batch} + if err := largepayload.TransformOrchestratorRequest(ctx, w.options.largePayloads, payloadRequest); err != nil { + w.logger.Errorf("%s: failed to hydrate session batch payloads: %v", turn.instanceID, err) + session.Close() + w.cleanupSessionWorkItem(ctx, scope, turn.instanceID, "release", releaseOrchestrationOnce(client, turn.token)) + return + } + sessionBytes += historySize(nil, batch) + results, next, err := session.Resume(ctx, batch, int(historyBase)) + if err != nil { + w.logger.Errorf("%s: failed to run session batch: %v", turn.instanceID, err) + session.Close() + w.cleanupSessionWorkItem(ctx, scope, turn.instanceID, "release", releaseOrchestrationOnce(client, turn.token)) + return + } + session = next + var ok bool + if response, ok = w.orchestratorResponse(ctx, client, turn, results, nil); !ok { + if session != nil { + session.Close() + } + return + } + } +} + +// canContinueOrchestrationSession reports whether a response may keep the +// session. A worker-built failure, for example for an oversized response, +// ends it. +func canContinueOrchestrationSession(response *protos.OrchestratorResponse) bool { + if response.NumEventsProcessed != nil { + return false + } + for _, action := range response.Actions { + if action.GetCompleteOrchestration() != nil { + return false + } + } + return true +} + +func historySize(eventLists ...[]*protos.HistoryEvent) int64 { + var size int64 + for _, events := range eventLists { + for _, event := range events { + size += int64(proto.Size(event)) + } } + return size } // failOversizedOrchestratorResponse converts a response that modern DTS cannot @@ -394,7 +578,10 @@ func populateOrchestratorActionTraceContexts( pastEvents []*protos.HistoryEvent, newEvents []*protos.HistoryEvent, ) error { - parent := executionStartedTraceContext(pastEvents, newEvents) + return populateActionTraceContexts(actions, executionStartedTraceContext(pastEvents, newEvents)) +} + +func populateActionTraceContexts(actions []*protos.OrchestratorAction, parent *protos.TraceContext) error { if parent == nil { return nil } @@ -605,44 +792,74 @@ func (w *TaskHubGrpcWorker) processEntityV2( completionToken string, request *protos.EntityRequest, ) { + w.processEntityWorkItem(ctx, client, nil, completionToken, request) +} + +// processEntityWorkItem runs a V2 entity work item. When scope allows +// extended sessions, the entity state stays cached for later batches. +func (w *TaskHubGrpcWorker) processEntityWorkItem( + ctx context.Context, + client protos.TaskHubSidecarServiceClient, + scope *workerSessionScope, + completionToken string, + request *protos.EntityRequest, +) { + result, ok := w.entityBatchResult(ctx, client, completionToken, request) + if !ok { + return + } + if result.FailureDetails == nil && scope.available() { + w.runEntitySession(ctx, client, scope, request.GetInstanceId(), completionToken, result) + return + } + w.completeEntityBatchResult(ctx, client, request.GetInstanceId(), result) +} + +// entityBatchResult runs one entity batch and returns the result to send. +// It returns false after it abandoned the work item. +func (w *TaskHubGrpcWorker) entityBatchResult( + ctx context.Context, + client protos.TaskHubSidecarServiceClient, + completionToken string, + request *protos.EntityRequest, +) (*protos.EntityBatchResult, bool) { batch, operationInfos, err := entityBatchFromRequestV2(request) if err != nil { w.logger.Errorf("invalid V2 entity work item: %v", err) w.abandonEntity(ctx, client, completionToken) - return + return nil, false } entityID, parseErr := api.EntityIDFromString(batch.GetInstanceId()) if parseErr != nil { w.logger.Errorf("%s: invalid entity instance ID: %v", batch.GetInstanceId(), parseErr) w.abandonEntity(ctx, client, completionToken) - return + return nil, false } if !matchesEntityWorkItemFilters(w.options.workItemFilters, entityID.Name) { w.logger.Warnf("%s: entity work item does not match configured filters; abandoning it", request.GetInstanceId()) w.abandonEntity(ctx, client, completionToken) - return + return nil, false } if err := largepayload.TransformEntityBatchRequest(ctx, w.options.largePayloads, batch); err != nil { w.logger.Errorf("%s: failed to hydrate entity batch payloads: %v", request.GetInstanceId(), err) if entityProcessingCanceled(ctx, err) { w.abandonEntity(ctx, client, completionToken) - return + return nil, false } - w.completeEntityBatchFailure(ctx, client, request.GetInstanceId(), completionToken, err) - return + return newEntityBatchFailure(completionToken, err), true } executor, ok := w.executor.(task.EntityExecutor) if !ok { w.logger.Error("task executor does not support entity work items") w.abandonEntity(ctx, client, completionToken) - return + return nil, false } result, err := executeEntitySafely(ctx, executor, batch) if err != nil { if entityProcessingCanceled(ctx, err) { w.logger.Warnf("%s: entity execution canceled; abandoning work item", request.GetInstanceId()) w.abandonEntity(ctx, client, completionToken) - return + return nil, false } w.logger.Errorf("%s: entity execution failed: %v", request.GetInstanceId(), err) result = newEntityBatchFailure(completionToken, err) @@ -660,13 +877,81 @@ func (w *TaskHubGrpcWorker) processEntityV2( w.logger.Errorf("%s: failed to externalize entity batch payloads: %v", request.GetInstanceId(), err) if entityProcessingCanceled(ctx, err) { w.abandonEntity(ctx, client, completionToken) + return nil, false + } + return newEntityBatchFailure(completionToken, err), true + } + return result, true +} + +// runEntitySession keeps an entity's state cached while the service sends new +// operation batches. Each batch runs with the state of the last accepted +// result. A failed batch ends the session with the unary RPC, and the service +// then abandons the work item. +func (w *TaskHubGrpcWorker) runEntitySession( + ctx context.Context, + client protos.TaskHubSidecarServiceClient, + scope *workerSessionScope, + instanceID string, + completionToken string, + result *protos.EntityBatchResult, +) { + lease := scope.run.sessions.acquire(entitySessionKind) + defer lease.release() + streaming := lease != nil + var ( + turnCount int32 + state *wrapperspb.StringValue + ) + for { + if !streaming || !scope.available() || result.FailureDetails != nil { + w.completeEntityBatchResultWithCount(ctx, client, instanceID, result, turnCount) + return + } + outcome, batch := w.awaitSessionBatch(ctx, scope, lease, sessionCall{ + instanceID: instanceID, + open: func(callCtx context.Context) (grpc.ServerStreamingClient[protos.CompleteTaskWithExtendedSessionResponse], error) { + result.CompletionToken = completionToken + result.CompletionCount = turnCount + return client.CompleteEntityTaskWithExtendedSession(callCtx, result) + }, + onAck: func(*protos.ExtendedSessionAck) int64 { + state = result.EntityState + turnCount++ + return int64(len(state.GetValue())) + }, + release: releaseEntityOnce(client, completionToken), + abandon: abandonEntityOnce(client, completionToken), + }) + switch outcome { + case sessionUseUnary: + streaming = false + continue + case sessionEnded: + return + } + var ok bool + result, ok = w.entityBatchResult(ctx, client, completionToken, &protos.EntityRequest{ + InstanceId: instanceID, + EntityState: state, + OperationRequests: entityOperationEvents(batch), + }) + if !ok { return } - w.completeEntityBatchFailure(ctx, client, request.GetInstanceId(), completionToken, err) - return } +} - w.completeEntityBatchResult(ctx, client, request.GetInstanceId(), result) +// entityOperationEvents drops the control events that may start a session +// batch. Entity batches contain only operation requests. +func entityOperationEvents(batch []*protos.HistoryEvent) []*protos.HistoryEvent { + operations := make([]*protos.HistoryEvent, 0, len(batch)) + for _, event := range batch { + if event.GetOrchestratorStarted() == nil { + operations = append(operations, event) + } + } + return operations } func executeEntitySafely( @@ -705,16 +990,6 @@ func newEntityBatchFailure(completionToken string, err error) *protos.EntityBatc } } -func (w *TaskHubGrpcWorker) completeEntityBatchFailure( - ctx context.Context, - client protos.TaskHubSidecarServiceClient, - instanceID string, - completionToken string, - err error, -) { - w.completeEntityBatchResult(ctx, client, instanceID, newEntityBatchFailure(completionToken, err)) -} - func validateEntityBatchResult( result *protos.EntityBatchResult, operationInfos []*protos.OperationInfo, @@ -756,6 +1031,20 @@ func (w *TaskHubGrpcWorker) completeEntityBatchResult( instanceID string, result *protos.EntityBatchResult, ) { + w.completeEntityBatchResultWithCount(ctx, client, instanceID, result, 0) +} + +// completeEntityBatchResultWithCount sends an entity result with the unary +// RPC. The count is set right before sending, because a failure result is a +// new message. +func (w *TaskHubGrpcWorker) completeEntityBatchResultWithCount( + ctx context.Context, + client protos.TaskHubSidecarServiceClient, + instanceID string, + result *protos.EntityBatchResult, + completionCount int32, +) { + result.CompletionCount = completionCount err := w.executeRPCWithRetry(ctx, "complete entity task", func(callCtx context.Context) error { _, callErr := client.CompleteEntityTask(callCtx, result) return callErr diff --git a/client/grpc_worker_session.go b/client/grpc_worker_session.go new file mode 100644 index 00000000..3b44c9a8 --- /dev/null +++ b/client/grpc_worker_session.go @@ -0,0 +1,489 @@ +package client + +import ( + "container/list" + "context" + "errors" + "fmt" + "io" + "sync" + "time" + + "github.com/microsoft/durabletask-go/internal/protos" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// DefaultExtendedSessionTimeout is the default idle and transfer timeout of an +// extended session. +const DefaultExtendedSessionTimeout = 30 * time.Second + +// defaultSessionCleanupTimeout bounds all release or abandon attempts for one +// session work item. +const defaultSessionCleanupTimeout = 30 * time.Second + +// errSessionCleanupFailed ends an intake generation after a session work item +// could not be released or abandoned. Closing intake makes the service abandon +// every work item of that generation. +var errSessionCleanupFailed = errors.New("extended-session cleanup failed; reconnecting the work item stream") + +// ExtendedSessionOptions configures extended sessions. In an extended session +// the worker keeps an orchestration or entity loaded in memory after a turn, +// and the service sends only the new messages for the next turn. This saves +// the replay of the full history. +// +// A waiting session keeps its concurrency slot, a service credit, and an +// open stream. Leave sessions off for hosts that scale to zero and behind +// proxies that close idle streams quickly. +type ExtendedSessionOptions struct { + // IdleTimeout is the longest time a session waits for new messages after + // the service accepts a turn. Zero means [DefaultExtendedSessionTimeout]. + IdleTimeout time.Duration + // TransferTimeout is the longest time from the first part of a batch to + // the end of the batch. Zero means [DefaultExtendedSessionTimeout]. + TransferTimeout time.Duration + // MaxOrchestrationSessions limits waiting orchestration sessions. Zero + // means half of the orchestration concurrency limit, rounded down. The + // value must be lower than that limit, so new work always has a slot. + MaxOrchestrationSessions int + // MaxEntitySessions limits waiting entity sessions in the same way. + MaxEntitySessions int + // MaxCachedBytes is an estimated memory budget for loaded sessions. When + // it is exceeded, the least recently used waiting sessions are ended. + // Zero means no byte limit. + MaxCachedBytes int64 +} + +// WithExtendedSessions turns on extended sessions. Sessions are off unless +// this option is set. When the service does not support sessions, the worker +// completes work items without them. +func WithExtendedSessions(options ExtendedSessionOptions) TaskHubGrpcWorkerOption { + return func(target *taskHubGrpcWorkerOptions) error { + if options.IdleTimeout < 0 || options.TransferTimeout < 0 { + return fmt.Errorf("extended session timeouts cannot be negative") + } + if options.MaxOrchestrationSessions < 0 || options.MaxEntitySessions < 0 { + return fmt.Errorf("extended session limits cannot be negative") + } + if options.MaxCachedBytes < 0 { + return fmt.Errorf("extended session cached bytes cannot be negative") + } + if options.IdleTimeout == 0 { + options.IdleTimeout = DefaultExtendedSessionTimeout + } + if options.TransferTimeout == 0 { + options.TransferTimeout = DefaultExtendedSessionTimeout + } + target.extendedSessions = &options + return nil + } +} + +// resolveExtendedSessionLimits applies the default session limits after all +// concurrency options are known. +func (options *taskHubGrpcWorkerOptions) resolveExtendedSessionLimits() error { + sessions := options.extendedSessions + if sessions == nil { + return nil + } + resolve := func(kind string, limit *int, concurrency int) error { + if *limit == 0 { + *limit = concurrency / 2 + return nil + } + if *limit >= concurrency { + return fmt.Errorf("maximum %s sessions must be lower than the %s concurrency limit %d", kind, kind, concurrency) + } + return nil + } + if err := resolve("orchestration", &sessions.MaxOrchestrationSessions, options.maxConcurrentOrchestrations); err != nil { + return err + } + return resolve("entity", &sessions.MaxEntitySessions, options.maxConcurrentEntities) +} + +type sessionKind int + +const ( + orchestrationSessionKind sessionKind = iota + entitySessionKind +) + +// sessionManager enforces the session limits of one worker run. +type sessionManager struct { + options *ExtendedSessionOptions + + mu sync.Mutex + orchestrations int + entities int + bytes int64 + idle *list.List +} + +// sessionLease is one session's share of the limits. Only the goroutine that +// owns the session uses it, except for eviction. +type sessionLease struct { + manager *sessionManager + kind sessionKind + bytes int64 + idle *list.Element + evict chan struct{} + evicted bool + done bool +} + +func newSessionManager(options *ExtendedSessionOptions) *sessionManager { + return &sessionManager{options: options, idle: list.New()} +} + +func (m *sessionManager) acquire(kind sessionKind) *sessionLease { + m.mu.Lock() + defer m.mu.Unlock() + count, limit := &m.orchestrations, m.options.MaxOrchestrationSessions + if kind == entitySessionKind { + count, limit = &m.entities, m.options.MaxEntitySessions + } + if *count >= limit { + return nil + } + *count++ + return &sessionLease{manager: m, kind: kind, evict: make(chan struct{})} +} + +// setIdle marks the session as waiting with the given estimated size. It +// ends the least recently used waiting sessions while over the byte budget. +func (l *sessionLease) setIdle(bytes int64) { + m := l.manager + m.mu.Lock() + defer m.mu.Unlock() + if l.evicted { + return + } + m.bytes += bytes - l.bytes + l.bytes = bytes + if l.idle == nil { + l.idle = m.idle.PushBack(l) + } + for m.options.MaxCachedBytes > 0 && m.bytes > m.options.MaxCachedBytes && m.idle.Len() > 0 { + victim := m.idle.Remove(m.idle.Front()).(*sessionLease) + victim.idle = nil + victim.evicted = true + m.bytes -= victim.bytes + victim.bytes = 0 + close(victim.evict) + } +} + +// setBusy marks the session as running, so it cannot be evicted. +func (l *sessionLease) setBusy() { + m := l.manager + m.mu.Lock() + defer m.mu.Unlock() + if l.idle != nil { + m.idle.Remove(l.idle) + l.idle = nil + } +} + +// release returns the lease. It is safe to call on a nil lease. +func (l *sessionLease) release() { + if l == nil { + return + } + m := l.manager + m.mu.Lock() + defer m.mu.Unlock() + if l.done { + return + } + l.done = true + if l.idle != nil { + m.idle.Remove(l.idle) + l.idle = nil + } + m.bytes -= l.bytes + l.bytes = 0 + if l.kind == entitySessionKind { + m.entities-- + } else { + m.orchestrations-- + } +} + +// workerSessionScope is the intake generation that dispatched a work item. +// Sessions belong to it: when its stream ends, the service abandons the work +// items, so the sessions close without a release. +type workerSessionScope struct { + run *grpcWorkerRun + connection *grpcWorkerConnection +} + +// available reports whether a new session turn may use the streaming RPC. +func (s *workerSessionScope) available() bool { + return s != nil && s.run.sessions != nil && + !s.connection.sessionsUnsupported.Load() && + s.run.intakeCtx.Err() == nil && + s.connection.streamCtx.Err() == nil +} + +type sessionOutcome int + +const ( + // sessionBatchReady means the turn was accepted and a full batch arrived. + sessionBatchReady sessionOutcome = iota + // sessionUseUnary means nothing was applied; complete the same turn with + // the unary RPC and the same count. + sessionUseUnary + // sessionEnded means the session is over and the work item needs no more + // calls from this worker. + sessionEnded +) + +type sessionTimerKind int + +const ( + sessionAckTimer sessionTimerKind = iota + sessionIdleTimer + sessionTransferTimer +) + +// sessionCall describes one turn of a session completion. +type sessionCall struct { + instanceID string + open func(context.Context) (grpc.ServerStreamingClient[protos.CompleteTaskWithExtendedSessionResponse], error) + // onAck runs when the service accepts the turn. It returns the estimated + // size of the loaded session. + onAck func(*protos.ExtendedSessionAck) int64 + release func(context.Context) error + abandon func(context.Context) error +} + +type sessionMessage struct { + response *protos.CompleteTaskWithExtendedSessionResponse + err error +} + +// awaitSessionBatch completes one turn with the streaming RPC and waits for +// the next batch. Transient errors before the acknowledgement retry the same +// request. It releases or abandons the work item itself when the session +// cannot continue. +func (w *TaskHubGrpcWorker) awaitSessionBatch( + ctx context.Context, + scope *workerSessionScope, + lease *sessionLease, + call sessionCall, +) (sessionOutcome, []*protos.HistoryEvent) { + for attempt := 1; ; attempt++ { + outcome, batch, preAckErr := w.runSessionStream(ctx, scope, lease, call) + if preAckErr == nil { + return outcome, batch + } + switch { + case errors.Is(preAckErr, errSessionStreamBudgetFull): + return sessionUseUnary, nil + case status.Code(preAckErr) == codes.Unimplemented: + w.logger.Infof("%s: the service does not support extended sessions; completing without them until the next reconnect", call.instanceID) + scope.connection.sessionsUnsupported.Store(true) + return sessionUseUnary, nil + case ctx.Err() != nil || scope.connection.streamCtx.Err() != nil: + return sessionEnded, nil + case isTransientWorkerRPCError(preAckErr) && attempt < w.options.transientRetryMaxAttempts: + if w.wait(ctx, w.retryDelay(attempt)) != nil { + return sessionEnded, nil + } + default: + w.logger.Warnf("%s: extended-session completion failed before the service accepted it; abandoning the work item: %v", call.instanceID, preAckErr) + w.cleanupSessionWorkItem(ctx, scope, call.instanceID, "abandon", call.abandon) + return sessionEnded, nil + } + } +} + +// runSessionStream runs one streaming completion attempt. This goroutine owns +// the stream and all of its timers, so a timeout and a message cannot both +// win. A non-nil error means that the attempt failed before the service +// accepted the turn. +func (w *TaskHubGrpcWorker) runSessionStream( + ctx context.Context, + scope *workerSessionScope, + lease *sessionLease, + call sessionCall, +) (sessionOutcome, []*protos.HistoryEvent, error) { + streamCtx, cancel := context.WithCancel(ctx) + defer cancel() + stream, err := call.open(streamCtx) + if err != nil { + return 0, nil, err + } + messages := make(chan sessionMessage) + received := make(chan struct{}) + go func() { + defer close(received) + for { + response, recvErr := stream.Recv() + select { + case messages <- sessionMessage{response: response, err: recvErr}: + case <-streamCtx.Done(): + return + } + if recvErr != nil { + return + } + } + }() + defer func() { + cancel() + <-received + }() + + timer, stopTimer := w.sessionTimer(sessionAckTimer, w.options.rpcTimeout) + defer func() { stopTimer() }() + restartTimer := func(kind sessionTimerKind, timeout time.Duration) { + stopTimer() + timer, stopTimer = w.sessionTimer(kind, timeout) + } + end := func(reason string, release bool) (sessionOutcome, []*protos.HistoryEvent, error) { + cancel() + w.logger.Debugf("%s: extended session ended: %s", call.instanceID, reason) + if release { + w.cleanupSessionWorkItem(ctx, scope, call.instanceID, "release", call.release) + } + return sessionEnded, nil, nil + } + + acked, transferring := false, false + var batch []*protos.HistoryEvent + for { + // Shutdown and eviction end a session only while it waits. + var stopping, evicted <-chan struct{} + if acked && !transferring { + stopping, evicted = scope.run.intakeCtx.Done(), lease.evict + } + select { + case <-ctx.Done(): + return end("the worker stopped", false) + case <-scope.connection.streamCtx.Done(): + return end("the work item stream ended", false) + case <-stopping: + return end("the worker is shutting down", true) + case <-evicted: + return end("the session cache is full", true) + case <-timer: + switch { + case !acked: + return 0, nil, status.Error(codes.DeadlineExceeded, "the service did not acknowledge the completion in time") + case transferring: + return end("the batch transfer timed out", true) + default: + return end("no new messages arrived before the idle timeout", true) + } + case message := <-messages: + if message.err != nil { + switch { + case !acked && errors.Is(message.err, io.EOF): + return 0, nil, status.Error(codes.Unknown, "the session stream ended before the acknowledgement") + case !acked: + return 0, nil, message.err + case transferring && errors.Is(message.err, io.EOF): + return sessionBatchReady, batch, nil + case errors.Is(message.err, io.EOF): + return end("the service ended the session", false) + default: + return end(fmt.Sprintf("the session stream failed: %v", message.err), true) + } + } + switch response := message.response.GetResponse().(type) { + case *protos.CompleteTaskWithExtendedSessionResponse_Ack: + if acked { + return end("the service sent a second acknowledgement", true) + } + acked = true + lease.setIdle(call.onAck(response.Ack)) + restartTimer(sessionIdleTimer, w.options.extendedSessions.IdleTimeout) + case *protos.CompleteTaskWithExtendedSessionResponse_HealthPing: + // A ping shows only that the connection is alive. + case *protos.CompleteTaskWithExtendedSessionResponse_NewMessages: + if !acked { + return 0, nil, status.Error(codes.Internal, "the service sent messages before the acknowledgement") + } + if !transferring { + transferring = true + lease.setBusy() + restartTimer(sessionTransferTimer, w.options.extendedSessions.TransferTimeout) + } + batch = append(batch, response.NewMessages.GetEvents()...) + default: + if !acked { + return 0, nil, status.Error(codes.Internal, "the service sent an unknown session message") + } + return end("the service sent an unknown session message", true) + } + } + } +} + +func (w *TaskHubGrpcWorker) sessionTimer(kind sessionTimerKind, timeout time.Duration) (<-chan time.Time, func()) { + if w.options.sessionTimerFn != nil { + return w.options.sessionTimerFn(kind, timeout), func() {} + } + timer := time.NewTimer(timeout) + return timer.C, func() { timer.Stop() } +} + +// cleanupSessionWorkItem releases or abandons a session work item. Attempts +// stop at one absolute deadline. If that passes, the worker ends the intake +// stream, so the service abandons all work items of this generation. +func (w *TaskHubGrpcWorker) cleanupSessionWorkItem( + ctx context.Context, + scope *workerSessionScope, + instanceID string, + operation string, + action func(context.Context) error, +) { + cleanupCtx, cancel := context.WithTimeout(ctx, w.options.sessionCleanupTimeout) + defer cancel() + for attempt := 1; ; attempt++ { + callCtx, cancelCall := context.WithTimeout(cleanupCtx, w.options.rpcTimeout) + err := action(callCtx) + cancelCall() + if err == nil || isWorkItemGone(err) || ctx.Err() != nil || scope.connection.streamCtx.Err() != nil { + return + } + if isTransientWorkerRPCError(err) && cleanupCtx.Err() == nil && + w.wait(cleanupCtx, w.retryDelay(attempt)) == nil && cleanupCtx.Err() == nil { + continue + } + w.logger.Errorf("%s: failed to %s the session work item; reconnecting the work item stream: %v", instanceID, operation, err) + scope.connection.abortForSessionCleanup() + return + } +} + +func releaseOrchestrationOnce(client protos.TaskHubSidecarServiceClient, token string) func(context.Context) error { + return func(ctx context.Context) error { + _, err := client.ReleaseTaskOrchestratorWorkItem(ctx, &protos.ReleaseTaskWorkItemRequest{CompletionToken: token}) + return err + } +} + +func releaseEntityOnce(client protos.TaskHubSidecarServiceClient, token string) func(context.Context) error { + return func(ctx context.Context) error { + _, err := client.ReleaseTaskEntityWorkItem(ctx, &protos.ReleaseTaskWorkItemRequest{CompletionToken: token}) + return err + } +} + +func abandonOrchestrationOnce(client protos.TaskHubSidecarServiceClient, token string) func(context.Context) error { + return func(ctx context.Context) error { + _, err := client.AbandonTaskOrchestratorWorkItem(ctx, &protos.AbandonOrchestrationTaskRequest{CompletionToken: token}) + return err + } +} + +func abandonEntityOnce(client protos.TaskHubSidecarServiceClient, token string) func(context.Context) error { + return func(ctx context.Context) error { + _, err := client.AbandonTaskEntityWorkItem(ctx, &protos.AbandonEntityTaskRequest{CompletionToken: token}) + return err + } +} diff --git a/client/grpc_worker_session_test.go b/client/grpc_worker_session_test.go new file mode 100644 index 00000000..c74d44f1 --- /dev/null +++ b/client/grpc_worker_session_test.go @@ -0,0 +1,961 @@ +package client + +import ( + "context" + "errors" + "io" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/helpers" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/microsoft/durabletask-go/task" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +// fakeSessionStream is a session completion stream that the test drives. +type fakeSessionStream struct { + grpc.ServerStreamingClient[protos.CompleteTaskWithExtendedSessionResponse] + ctx context.Context + messages chan sessionMessage +} + +func (s *fakeSessionStream) Recv() (*protos.CompleteTaskWithExtendedSessionResponse, error) { + select { + case <-s.ctx.Done(): + return nil, status.FromContextError(s.ctx.Err()).Err() + case message := <-s.messages: + return message.response, message.err + } +} + +func (s *fakeSessionStream) ack(count int32) { + s.messages <- sessionMessage{response: &protos.CompleteTaskWithExtendedSessionResponse{ + Response: &protos.CompleteTaskWithExtendedSessionResponse_Ack{Ack: &protos.ExtendedSessionAck{HistoryEventCount: count}}, + }} +} + +func (s *fakeSessionStream) ping() { + s.messages <- sessionMessage{response: &protos.CompleteTaskWithExtendedSessionResponse{ + Response: &protos.CompleteTaskWithExtendedSessionResponse_HealthPing{HealthPing: &protos.HealthPing{}}, + }} +} + +func (s *fakeSessionStream) chunk(events ...*protos.HistoryEvent) { + s.messages <- sessionMessage{response: &protos.CompleteTaskWithExtendedSessionResponse{ + Response: &protos.CompleteTaskWithExtendedSessionResponse_NewMessages{NewMessages: &protos.HistoryChunk{Events: events}}, + }} +} + +func (s *fakeSessionStream) end(err error) { + if err == nil { + err = io.EOF + } + s.messages <- sessionMessage{err: err} +} + +// batch sends a complete batch and ends the stream with OK. +func (s *fakeSessionStream) batch(events ...*protos.HistoryEvent) { + s.chunk(events...) + s.end(nil) +} + +type fakeSessionCall struct { + orchestration *protos.OrchestratorResponse + entity *protos.EntityBatchResult + stream *fakeSessionStream +} + +func (c *fakeSessionCall) count() int32 { + if c.entity != nil { + return c.entity.CompletionCount + } + return c.orchestration.CompletionCount +} + +// sessionSchedulerClient adds the extended-session RPCs to the fake scheduler. +type sessionSchedulerClient struct { + *fakeSchedulerClient + calls chan *fakeSessionCall + openErr func() error + releaseErr func() error + abandonErr func() error + + releases atomic.Int32 + orchestrationAbort atomic.Int32 + connects atomic.Int32 +} + +func newSessionSchedulerClient() *sessionSchedulerClient { + return &sessionSchedulerClient{ + fakeSchedulerClient: &fakeSchedulerClient{stream: newFakeWorkItemStream(16)}, + calls: make(chan *fakeSessionCall, 16), + } +} + +func (c *sessionSchedulerClient) GetWorkItems( + ctx context.Context, + request *protos.GetWorkItemsRequest, + opts ...grpc.CallOption, +) (protos.TaskHubSidecarService_GetWorkItemsClient, error) { + c.connects.Add(1) + return c.fakeSchedulerClient.GetWorkItems(ctx, request, opts...) +} + +func (c *sessionSchedulerClient) open(ctx context.Context, call *fakeSessionCall) (grpc.ServerStreamingClient[protos.CompleteTaskWithExtendedSessionResponse], error) { + if c.openErr != nil { + if err := c.openErr(); err != nil { + return nil, err + } + } + call.stream = &fakeSessionStream{ctx: ctx, messages: make(chan sessionMessage, 16)} + c.calls <- call + return call.stream, nil +} + +func (c *sessionSchedulerClient) CompleteOrchestratorTaskWithExtendedSession( + ctx context.Context, + response *protos.OrchestratorResponse, + _ ...grpc.CallOption, +) (grpc.ServerStreamingClient[protos.CompleteTaskWithExtendedSessionResponse], error) { + return c.open(ctx, &fakeSessionCall{orchestration: proto.Clone(response).(*protos.OrchestratorResponse)}) +} + +func (c *sessionSchedulerClient) CompleteEntityTaskWithExtendedSession( + ctx context.Context, + result *protos.EntityBatchResult, + _ ...grpc.CallOption, +) (grpc.ServerStreamingClient[protos.CompleteTaskWithExtendedSessionResponse], error) { + return c.open(ctx, &fakeSessionCall{entity: proto.Clone(result).(*protos.EntityBatchResult)}) +} + +func (c *sessionSchedulerClient) release() (*protos.ReleaseTaskWorkItemResponse, error) { + c.releases.Add(1) + if c.releaseErr != nil { + if err := c.releaseErr(); err != nil { + return nil, err + } + } + return &protos.ReleaseTaskWorkItemResponse{}, nil +} + +func (c *sessionSchedulerClient) ReleaseTaskOrchestratorWorkItem( + context.Context, + *protos.ReleaseTaskWorkItemRequest, + ...grpc.CallOption, +) (*protos.ReleaseTaskWorkItemResponse, error) { + return c.release() +} + +func (c *sessionSchedulerClient) ReleaseTaskEntityWorkItem( + context.Context, + *protos.ReleaseTaskWorkItemRequest, + ...grpc.CallOption, +) (*protos.ReleaseTaskWorkItemResponse, error) { + return c.release() +} + +func (c *sessionSchedulerClient) AbandonTaskOrchestratorWorkItem( + ctx context.Context, + request *protos.AbandonOrchestrationTaskRequest, + opts ...grpc.CallOption, +) (*protos.AbandonOrchestrationTaskResponse, error) { + c.orchestrationAbort.Add(1) + if c.abandonErr != nil { + if err := c.abandonErr(); err != nil { + return nil, err + } + } + return c.fakeSchedulerClient.AbandonTaskOrchestratorWorkItem(ctx, request, opts...) +} + +func (c *sessionSchedulerClient) nextCall(t *testing.T) *fakeSessionCall { + t.Helper() + select { + case call := <-c.calls: + return call + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for a session completion") + return nil + } +} + +func (c *sessionSchedulerClient) noCall(t *testing.T) { + t.Helper() + select { + case call := <-c.calls: + t.Fatalf("unexpected session completion with count %d", call.count()) + default: + } +} + +func (c *sessionSchedulerClient) unaryOrchestrations() []*protos.OrchestratorResponse { + c.mu.Lock() + defer c.mu.Unlock() + return append([]*protos.OrchestratorResponse(nil), c.orchestrationCompletions...) +} + +func (c *sessionSchedulerClient) unaryEntities() []*protos.EntityBatchResult { + c.mu.Lock() + defer c.mu.Unlock() + return append([]*protos.EntityBatchResult(nil), c.entityCompletions...) +} + +func (c *sessionSchedulerClient) abandons() int { + c.mu.Lock() + defer c.mu.Unlock() + return c.orchestrationAbandons +} + +// fakeSessionTimers lets a test fire session timers by hand. +type fakeSessionTimers struct { + mu sync.Mutex + created []sessionTimerKind + timers chan fakeSessionTimer +} + +type fakeSessionTimer struct { + kind sessionTimerKind + fire chan time.Time +} + +func newFakeSessionTimers() *fakeSessionTimers { + return &fakeSessionTimers{timers: make(chan fakeSessionTimer, 64)} +} + +func (f *fakeSessionTimers) new(kind sessionTimerKind, _ time.Duration) <-chan time.Time { + timer := fakeSessionTimer{kind: kind, fire: make(chan time.Time, 1)} + f.mu.Lock() + f.created = append(f.created, kind) + f.mu.Unlock() + f.timers <- timer + return timer.fire +} + +// next returns the next created timer of the given kind. +func (f *fakeSessionTimers) next(t *testing.T, kind sessionTimerKind) fakeSessionTimer { + t.Helper() + deadline := time.After(5 * time.Second) + for { + select { + case timer := <-f.timers: + if timer.kind == kind { + return timer + } + case <-deadline: + t.Fatalf("timed out waiting for session timer %d", kind) + } + } +} + +func (f *fakeSessionTimers) count(kind sessionTimerKind) int { + f.mu.Lock() + defer f.mu.Unlock() + n := 0 + for _, created := range f.created { + if created == kind { + n++ + } + } + return n +} + +type sessionWorkerHarness struct { + worker *TaskHubGrpcWorker + client *sessionSchedulerClient + timers *fakeSessionTimers + starts *atomic.Int32 + cancel context.CancelFunc +} + +func sessionOrchestrator(starts *atomic.Int32, activities int) task.Orchestrator { + return func(ctx *task.OrchestrationContext) (any, error) { + starts.Add(1) + results := make([]string, 0, activities) + for range activities { + var result string + if err := ctx.CallActivity("work").Await(&result); err != nil { + return nil, err + } + results = append(results, result) + } + return strings.Join(results, ","), nil + } +} + +func startSessionWorker( + t *testing.T, + registry *task.TaskRegistry, + client *sessionSchedulerClient, + opts ...TaskHubGrpcWorkerOption, +) *sessionWorkerHarness { + t.Helper() + options := []TaskHubGrpcWorkerOption{ + WithWorkerHelloTimeout(time.Second), + WithWorkerSilentDisconnectTimeout(10 * time.Second), + WithWorkerRPCTimeout(time.Second), + WithWorkerReconnectBackoff(time.Millisecond, 5*time.Millisecond), + WithWorkerTransientRetryPolicy(3, time.Millisecond, 5*time.Millisecond), + WithMaxConcurrentOrchestrationWorkItems(4), + WithMaxConcurrentEntityWorkItems(4), + WithExtendedSessions(ExtendedSessionOptions{}), + } + worker, err := newTaskHubGrpcWorker( + func(context.Context) (protos.TaskHubSidecarServiceClient, io.Closer, error) { + return client, nil, nil + }, + registry, + api.DefaultLogger(), + append(options, opts...)..., + ) + require.NoError(t, err) + timers := newFakeSessionTimers() + worker.options.sessionTimerFn = timers.new + ctx, cancel := context.WithCancel(context.Background()) + require.NoError(t, worker.Start(ctx)) + h := &sessionWorkerHarness{worker: worker, client: client, timers: timers, cancel: cancel} + t.Cleanup(func() { + cancel() + // A turn that a test never acknowledges waits for its acknowledgement + // timer, so end the worker quickly. + shutdownCtx, cancelShutdown := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancelShutdown() + _ = worker.Shutdown(shutdownCtx) + }) + return h +} + +func newSessionHarness(t *testing.T, activities int, opts ...TaskHubGrpcWorkerOption) *sessionWorkerHarness { + t.Helper() + starts := new(atomic.Int32) + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("session", sessionOrchestrator(starts, activities))) + h := startSessionWorker(t, registry, newSessionSchedulerClient(), opts...) + h.starts = starts + return h +} + +func (h *sessionWorkerHarness) dispatch(instanceID string, newEvents ...*protos.HistoryEvent) { + if len(newEvents) == 0 { + newEvents = []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("session", instanceID, nil, nil, nil, nil), + } + } + h.client.stream.results <- fakeWorkItemResult{item: &protos.WorkItem{ + Request: &protos.WorkItem_OrchestratorRequest{OrchestratorRequest: &protos.OrchestratorRequest{ + InstanceId: instanceID, + NewEvents: newEvents, + }}, + CompletionToken: "token-" + instanceID, + }} +} + +func taskCompleted(id int32, result string) []*protos.HistoryEvent { + return []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewTaskCompletedEvent(id, wrapperspb.String(`"`+result+`"`)), + } +} + +func (h *sessionWorkerHarness) waitUnary(t *testing.T, n int) []*protos.OrchestratorResponse { + t.Helper() + require.Eventually(t, func() bool { return len(h.client.unaryOrchestrations()) >= n }, 5*time.Second, time.Millisecond) + return h.client.unaryOrchestrations() +} + +func completedOutput(t *testing.T, response *protos.OrchestratorResponse) string { + t.Helper() + require.Len(t, response.Actions, 1) + completed := response.Actions[0].GetCompleteOrchestration() + require.NotNil(t, completed) + require.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, completed.OrchestrationStatus, + "failure: %v", completed.GetFailureDetails()) + return completed.GetResult().GetValue() +} + +func TestWorkerSessionRunsTurnsWithoutReplay(t *testing.T) { + h := newSessionHarness(t, 2) + h.dispatch("multi") + + first := h.client.nextCall(t) + require.EqualValues(t, 0, first.count()) + require.Equal(t, "token-multi", first.orchestration.CompletionToken) + require.Equal(t, "work", first.orchestration.Actions[0].GetScheduleTask().GetName()) + first.stream.ack(4) + first.stream.ping() + first.stream.batch(taskCompleted(0, "a")...) + + second := h.client.nextCall(t) + require.EqualValues(t, 1, second.count()) + require.Len(t, second.orchestration.Actions, 1) + require.EqualValues(t, 1, second.orchestration.Actions[0].Id) + second.stream.ack(8) + second.stream.batch(taskCompleted(1, "b")...) + + final := h.waitUnary(t, 1)[0] + require.EqualValues(t, 2, final.CompletionCount, "the terminal unary completion must carry the session count") + require.Equal(t, `"a,b"`, completedOutput(t, final)) + require.EqualValues(t, 1, h.starts.Load(), "the orchestrator must not replay between session turns") + require.Zero(t, h.client.releases.Load()) + require.Zero(t, h.client.abandons()) +} + +func TestWorkerSessionRetriesTransientErrorWithSameCount(t *testing.T) { + h := newSessionHarness(t, 2) + h.dispatch("retry") + + first := h.client.nextCall(t) + first.stream.ack(4) + first.stream.batch(taskCompleted(0, "a")...) + + failed := h.client.nextCall(t) + require.EqualValues(t, 1, failed.count()) + failed.stream.end(status.Error(codes.Unavailable, "lost")) + retried := h.client.nextCall(t) + require.EqualValues(t, 1, retried.count(), "a retry before the acknowledgement must reuse the turn count") + require.True(t, proto.Equal(failed.orchestration, retried.orchestration)) + retried.stream.ack(8) + retried.stream.batch(taskCompleted(1, "b")...) + + require.EqualValues(t, 2, h.waitUnary(t, 1)[0].CompletionCount) +} + +func TestWorkerSessionAbandonsOnPreAckFailure(t *testing.T) { + for _, err := range []error{ + status.Error(codes.NotFound, "count mismatch"), + status.Error(codes.InvalidArgument, "bad request"), + } { + t.Run(status.Code(err).String(), func(t *testing.T) { + h := newSessionHarness(t, 2) + h.dispatch("abandon") + first := h.client.nextCall(t) + first.stream.end(err) + require.Eventually(t, func() bool { return h.client.abandons() == 1 }, 5*time.Second, time.Millisecond) + h.client.noCall(t) + require.Empty(t, h.client.unaryOrchestrations()) + require.Zero(t, h.client.releases.Load()) + }) + } +} + +func TestWorkerSessionAckTimeoutRetries(t *testing.T) { + h := newSessionHarness(t, 1) + h.dispatch("ack-timeout") + first := h.client.nextCall(t) + h.timers.next(t, sessionAckTimer).fire <- time.Now() + retried := h.client.nextCall(t) + require.EqualValues(t, 0, retried.count()) + require.True(t, proto.Equal(first.orchestration, retried.orchestration)) +} + +func TestWorkerSessionReleasesAfterAckFailures(t *testing.T) { + tests := []struct { + name string + fail func(t *testing.T, h *sessionWorkerHarness, stream *fakeSessionStream) + }{ + {name: "stream breaks while idle", fail: func(_ *testing.T, _ *sessionWorkerHarness, stream *fakeSessionStream) { + stream.end(status.Error(codes.Unavailable, "broken")) + }}, + {name: "batch cut off after first chunk", fail: func(_ *testing.T, _ *sessionWorkerHarness, stream *fakeSessionStream) { + stream.chunk(taskCompleted(0, "partial")...) + stream.end(status.Error(codes.Unavailable, "cut")) + }}, + {name: "idle timeout", fail: func(t *testing.T, h *sessionWorkerHarness, stream *fakeSessionStream) { + stream.ping() + stream.ping() + h.timers.next(t, sessionIdleTimer).fire <- time.Now() + }}, + {name: "transfer timeout", fail: func(t *testing.T, h *sessionWorkerHarness, stream *fakeSessionStream) { + stream.chunk(taskCompleted(0, "partial")...) + h.timers.next(t, sessionTransferTimer).fire <- time.Now() + }}, + {name: "second acknowledgement", fail: func(_ *testing.T, _ *sessionWorkerHarness, stream *fakeSessionStream) { + stream.ack(1) + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + h := newSessionHarness(t, 2) + h.dispatch("release") + first := h.client.nextCall(t) + first.stream.ack(4) + tt.fail(t, h, first.stream) + require.Eventually(t, func() bool { return h.client.releases.Load() == 1 }, 5*time.Second, time.Millisecond) + // The partial batch is discarded: no further turn runs. + h.client.noCall(t) + require.Empty(t, h.client.unaryOrchestrations()) + require.Zero(t, h.client.abandons()) + require.EqualValues(t, 1, h.starts.Load()) + }) + } +} + +func TestWorkerSessionPingsDoNotResetIdleTimer(t *testing.T) { + h := newSessionHarness(t, 2) + h.dispatch("pings") + first := h.client.nextCall(t) + first.stream.ack(4) + for range 5 { + first.stream.ping() + } + first.stream.chunk(taskCompleted(0, "a")...) + // The transfer timer starts only after every earlier message was handled. + h.timers.next(t, sessionTransferTimer) + require.Equal(t, 1, h.timers.count(sessionIdleTimer), "pings must not restart the idle timer") + first.stream.end(nil) + require.EqualValues(t, 1, h.client.nextCall(t).count()) +} + +func TestWorkerSessionTimeoutAndFirstChunkHaveOneWinner(t *testing.T) { + h := newSessionHarness(t, 2) + h.dispatch("race") + first := h.client.nextCall(t) + first.stream.ack(4) + idle := h.timers.next(t, sessionIdleTimer) + first.stream.chunk(taskCompleted(0, "a")...) + transfer := h.timers.next(t, sessionTransferTimer) + // The old idle timer fires after the first chunk switched to transfer. + idle.fire <- time.Now() + first.stream.end(nil) + second := h.client.nextCall(t) + require.EqualValues(t, 1, second.count()) + select { + case transfer.fire <- time.Now(): + default: + } + require.Zero(t, h.client.releases.Load()) +} + +func TestWorkerSessionEndsWhenServiceEndsStream(t *testing.T) { + h := newSessionHarness(t, 2) + h.dispatch("service-end") + first := h.client.nextCall(t) + first.stream.ack(4) + first.stream.end(nil) + require.Never(t, func() bool { + return h.client.releases.Load() != 0 || h.client.abandons() != 0 || len(h.client.unaryOrchestrations()) != 0 + }, 50*time.Millisecond, 5*time.Millisecond) + h.client.noCall(t) +} + +func TestWorkerSessionUnimplementedFallsBackToUnary(t *testing.T) { + h := newSessionHarness(t, 1) + h.dispatch("first") + first := h.client.nextCall(t) + first.stream.end(status.Error(codes.Unimplemented, "no sessions")) + unary := h.waitUnary(t, 1)[0] + require.EqualValues(t, 0, unary.CompletionCount) + require.True(t, proto.Equal(first.orchestration, unary), "the fallback must send the same result") + + // Sessions stay off on this intake generation. + h.dispatch("second") + require.Len(t, h.waitUnary(t, 2), 2) + h.client.noCall(t) +} + +func TestWorkerSessionUnimplementedAfterFirstTurnUsesTurnCount(t *testing.T) { + h := newSessionHarness(t, 2) + h.dispatch("later") + first := h.client.nextCall(t) + first.stream.ack(4) + first.stream.batch(taskCompleted(0, "a")...) + second := h.client.nextCall(t) + second.stream.end(status.Error(codes.Unimplemented, "switched off")) + unary := h.waitUnary(t, 1)[0] + require.EqualValues(t, 1, unary.CompletionCount) + require.Equal(t, "work", unary.Actions[0].GetScheduleTask().GetName()) +} + +func TestWorkerSessionStreamBudgetFullUsesUnary(t *testing.T) { + h := newSessionHarness(t, 1) + h.client.openErr = func() error { return errSessionStreamBudgetFull } + h.dispatch("budget") + unary := h.waitUnary(t, 1)[0] + require.EqualValues(t, 0, unary.CompletionCount) + require.Len(t, unary.Actions, 1) +} + +func TestWorkerSessionLimitUsesUnary(t *testing.T) { + h := newSessionHarness(t, 1, WithExtendedSessions(ExtendedSessionOptions{MaxOrchestrationSessions: 1})) + h.dispatch("one") + first := h.client.nextCall(t) + first.stream.ack(3) + h.dispatch("two") + unary := h.waitUnary(t, 1)[0] + require.Equal(t, "two", unary.InstanceId) +} + +func TestWorkerSessionPartialTurnUsesUnary(t *testing.T) { + starts := new(atomic.Int32) + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("session", sessionOrchestrator(starts, 1))) + h := startSessionWorker(t, registry, newSessionSchedulerClient()) + h.worker.executor = task.NewTaskExecutor(registry, + task.WithOrchestrationOptions(task.OrchestrationOptions{MaxEventsPerTurn: 1})) + h.dispatch("partial", + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("session", "partial", nil, nil, nil, nil), + helpers.NewEventRaisedEvent("x", nil), + ) + unary := h.waitUnary(t, 1)[0] + require.EqualValues(t, 1, unary.GetNumEventsProcessed().GetValue()) + h.client.noCall(t) +} + +func TestWorkerSessionRebuiltResponsesCarryCount(t *testing.T) { + t.Run("oversized response", func(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("session", func(ctx *task.OrchestrationContext) (any, error) { + if err := ctx.CallActivity("work").Await(nil); err != nil { + return nil, err + } + return nil, ctx.CallActivity("big", task.WithRawActivityInput(strings.Repeat("x", 100*1024))).Await(nil) + })) + h := startSessionWorker(t, registry, newSessionSchedulerClient(), WithMaxOrchestratorCompletionBytes(64*1024)) + h.dispatch("oversized") + first := h.client.nextCall(t) + first.stream.ack(4) + first.stream.batch(taskCompleted(0, "a")...) + unary := h.waitUnary(t, 1)[0] + require.EqualValues(t, 1, unary.CompletionCount) + require.Equal(t, "token-oversized", unary.CompletionToken) + require.Equal(t, string(api.ErrorTypeOrchestratorResponseTooLarge), + unary.Actions[0].GetCompleteOrchestration().GetFailureDetails().GetErrorType()) + }) + t.Run("orchestrator panic", func(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("session", func(ctx *task.OrchestrationContext) (any, error) { + if err := ctx.CallActivity("work").Await(nil); err != nil { + return nil, err + } + panic("boom") + })) + h := startSessionWorker(t, registry, newSessionSchedulerClient()) + h.dispatch("panic") + first := h.client.nextCall(t) + first.stream.ack(4) + first.stream.batch(taskCompleted(0, "a")...) + unary := h.waitUnary(t, 1)[0] + require.EqualValues(t, 1, unary.CompletionCount) + require.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, + unary.Actions[0].GetCompleteOrchestration().GetOrchestrationStatus()) + }) +} + +func TestWorkerSessionInvalidBatchReleases(t *testing.T) { + h := newSessionHarness(t, 2) + h.dispatch("invalid") + first := h.client.nextCall(t) + first.stream.ack(4) + // A batch must start with OrchestratorStarted. + first.stream.batch(helpers.NewTaskCompletedEvent(0, nil)) + require.Eventually(t, func() bool { return h.client.releases.Load() == 1 }, 5*time.Second, time.Millisecond) + require.Empty(t, h.client.unaryOrchestrations()) +} + +func TestWorkerSessionCleanupFailureReconnectsIntake(t *testing.T) { + h := newSessionHarness(t, 2) + h.worker.options.sessionCleanupTimeout = 20 * time.Millisecond + h.client.releaseErr = func() error { return status.Error(codes.Unavailable, "down") } + h.dispatch("cleanup") + first := h.client.nextCall(t) + first.stream.ack(4) + first.stream.end(status.Error(codes.Unavailable, "broken")) + require.Eventually(t, func() bool { return h.client.connects.Load() >= 2 }, 5*time.Second, time.Millisecond, + "a failed cleanup must reconnect the work item stream") + require.Greater(t, h.client.releases.Load(), int32(1), "release must be retried before giving up") + require.True(t, h.worker.Running()) +} + +func TestWorkerSessionAbandonFailureReconnectsIntake(t *testing.T) { + h := newSessionHarness(t, 1) + h.worker.options.sessionCleanupTimeout = 20 * time.Millisecond + h.client.abandonErr = func() error { return status.Error(codes.Unavailable, "down") } + h.dispatch("abandon-fails") + first := h.client.nextCall(t) + first.stream.end(status.Error(codes.InvalidArgument, "rejected")) + require.Eventually(t, func() bool { return h.client.connects.Load() >= 2 }, 5*time.Second, time.Millisecond) + require.Greater(t, h.client.orchestrationAbort.Load(), int32(1)) +} + +func TestWorkerSessionClosesWithRetiredGeneration(t *testing.T) { + h := newSessionHarness(t, 2) + h.dispatch("retired") + first := h.client.nextCall(t) + first.stream.ack(4) + h.timers.next(t, sessionIdleTimer) + // The intake stream fails, so the service abandons the generation's work. + h.client.stream.results <- fakeWorkItemResult{err: status.Error(codes.Unavailable, "intake lost")} + require.Eventually(t, func() bool { return h.client.connects.Load() >= 2 }, 5*time.Second, time.Millisecond) + require.Eventually(t, func() bool { + select { + case <-first.stream.ctx.Done(): + return true + default: + return false + } + }, 5*time.Second, time.Millisecond, "the session stream must close with its generation") + require.Zero(t, h.client.releases.Load(), "a retired generation's sessions close without release") + require.Zero(t, h.client.abandons()) +} + +func TestWorkerSessionShutdownReleasesIdleSessions(t *testing.T) { + h := newSessionHarness(t, 2) + h.dispatch("shutdown") + first := h.client.nextCall(t) + first.stream.ack(4) + h.timers.next(t, sessionIdleTimer) + h.cancel() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + require.NoError(t, h.worker.Shutdown(ctx)) + require.EqualValues(t, 1, h.client.releases.Load()) +} + +func TestWorkerSessionEvictsLeastRecentlyUsedIdleSession(t *testing.T) { + manager := newSessionManager(&ExtendedSessionOptions{ + MaxOrchestrationSessions: 3, + MaxCachedBytes: 100, + }) + first := manager.acquire(orchestrationSessionKind) + second := manager.acquire(orchestrationSessionKind) + third := manager.acquire(orchestrationSessionKind) + require.Nil(t, manager.acquire(orchestrationSessionKind)) + first.setIdle(40) + second.setIdle(40) + third.setBusy() + require.Equal(t, int64(80), manager.bytes) + // Touching first makes second the least recently used session. + first.setBusy() + first.setIdle(40) + third.setIdle(40) + select { + case <-second.evict: + default: + t.Fatal("the least recently used idle session must be evicted") + } + select { + case <-first.evict: + t.Fatal("only one session should be evicted") + default: + } + require.Equal(t, int64(80), manager.bytes) + second.release() + second.release() + first.release() + third.release() + require.Zero(t, manager.bytes) + require.Zero(t, manager.orchestrations) +} + +func TestWorkerSessionEvictionReleasesWorkItem(t *testing.T) { + h := newSessionHarness(t, 2, WithExtendedSessions(ExtendedSessionOptions{MaxCachedBytes: 1})) + h.dispatch("evicted") + first := h.client.nextCall(t) + first.stream.ack(4) + require.Eventually(t, func() bool { return h.client.releases.Load() == 1 }, 5*time.Second, time.Millisecond) +} + +func entitySessionRequest(instanceID string, operations ...*protos.HistoryEvent) *protos.EntityRequest { + return &protos.EntityRequest{InstanceId: instanceID, OperationRequests: operations} +} + +func entitySignal(operation string) *protos.HistoryEvent { + return newFakeEntitySignalRequest(operation, nil).OperationRequests[0] +} + +func TestWorkerEntitySessionUsesCachedState(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddEntityN("counter", func(ctx *task.EntityContext) (any, error) { + var value int + if ctx.HasState() { + if err := ctx.GetState(&value); err != nil { + return nil, err + } + } + return nil, ctx.SetState(value + 1) + })) + client := newSessionSchedulerClient() + h := startSessionWorker(t, registry, client) + var requests []*protos.EntityBatchRequest + var mu sync.Mutex + base := h.worker.executor.(task.EntityExecutor) + h.worker.executor = &recordingExecutor{executeEntity: func(ctx context.Context, request *protos.EntityBatchRequest) (*protos.EntityBatchResult, error) { + mu.Lock() + requests = append(requests, proto.Clone(request).(*protos.EntityBatchRequest)) + mu.Unlock() + return base.ExecuteEntity(ctx, request) + }} + client.stream.results <- fakeWorkItemResult{item: &protos.WorkItem{ + Request: &protos.WorkItem_EntityRequestV2{EntityRequestV2: entitySessionRequest("@counter@a", entitySignal("add"))}, + CompletionToken: "entity-token", + }} + + first := client.nextCall(t) + require.EqualValues(t, 0, first.count()) + require.Equal(t, "1", first.entity.EntityState.GetValue()) + first.stream.ack(0) + first.stream.batch(helpers.NewOrchestratorStartedEvent(), entitySignal("add"), entitySignal("add")) + + second := client.nextCall(t) + require.EqualValues(t, 1, second.count()) + require.Equal(t, "3", second.entity.EntityState.GetValue()) + require.Len(t, second.entity.Results, 2) + second.stream.ack(0) + // A batch whose executor fails ends the session with the unary RPC. + h.worker.executor = &recordingExecutor{executeEntity: func(context.Context, *protos.EntityBatchRequest) (*protos.EntityBatchResult, error) { + return nil, errors.New("framework failure") + }} + second.stream.batch(entitySignal("add")) + + require.Eventually(t, func() bool { return len(client.unaryEntities()) == 1 }, 5*time.Second, time.Millisecond) + failed := client.unaryEntities()[0] + require.EqualValues(t, 2, failed.CompletionCount) + require.Equal(t, "entity-token", failed.CompletionToken) + require.NotNil(t, failed.FailureDetails) + + mu.Lock() + defer mu.Unlock() + require.Len(t, requests, 2) + require.Equal(t, "1", requests[1].EntityState.GetValue(), "a resumed batch must use the cached state") + require.Len(t, requests[1].Operations, 2, "control events are not entity operations") +} + +func TestWithExtendedSessionsValidation(t *testing.T) { + newWorker := func(opts ...TaskHubGrpcWorkerOption) error { + _, err := newTaskHubGrpcWorker( + func(context.Context) (protos.TaskHubSidecarServiceClient, io.Closer, error) { return nil, nil, nil }, + task.NewTaskRegistry(), + api.DefaultLogger(), + opts..., + ) + return err + } + require.Error(t, newWorker(WithExtendedSessions(ExtendedSessionOptions{IdleTimeout: -1}))) + require.Error(t, newWorker(WithExtendedSessions(ExtendedSessionOptions{MaxEntitySessions: -1}))) + require.Error(t, newWorker(WithExtendedSessions(ExtendedSessionOptions{MaxCachedBytes: -1}))) + require.ErrorContains(t, newWorker( + WithMaxConcurrentOrchestrationWorkItems(2), + WithExtendedSessions(ExtendedSessionOptions{MaxOrchestrationSessions: 2}), + ), "must be lower") + + options := defaultTaskHubGrpcWorkerOptions() + require.NoError(t, WithExtendedSessions(ExtendedSessionOptions{})(&options)) + options.maxConcurrentOrchestrations = 1 + options.maxConcurrentEntities = 9 + require.NoError(t, options.resolveExtendedSessionLimits()) + require.Zero(t, options.extendedSessions.MaxOrchestrationSessions, "one slot leaves no room for sessions") + require.Equal(t, 4, options.extendedSessions.MaxEntitySessions) + require.Equal(t, DefaultExtendedSessionTimeout, options.extendedSessions.IdleTimeout) + require.Equal(t, DefaultExtendedSessionTimeout, options.extendedSessions.TransferTimeout) +} + +func TestWorkerSessionsOffByDefault(t *testing.T) { + starts := new(atomic.Int32) + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("session", sessionOrchestrator(starts, 1))) + client := newSessionSchedulerClient() + worker := newFakeWorker(t, client.fakeSchedulerClient) + worker.clientFactory = func(context.Context) (protos.TaskHubSidecarServiceClient, io.Closer, error) { + return client, nil, nil + } + worker.executor = task.NewTaskExecutor(registry) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + require.NoError(t, worker.Start(ctx)) + h := &sessionWorkerHarness{worker: worker, client: client} + h.dispatch("off") + h.waitUnary(t, 1) + client.noCall(t) + cancel() + require.NoError(t, worker.Shutdown(context.Background())) +} + +// sessionStreamConn records session streams opened on a completion connection. +type sessionStreamConn struct { + fakeClientConn + opened atomic.Int32 +} + +func (c *sessionStreamConn) NewStream(ctx context.Context, desc *grpc.StreamDesc, method string, opts ...grpc.CallOption) (grpc.ClientStream, error) { + c.opened.Add(1) + return c.fakeClientConn.NewStream(ctx, desc, method, opts...) +} + +func TestWorkerCompletionTransportRoutesSessionStreams(t *testing.T) { + intake := &sessionStreamConn{} + completions := []*sessionStreamConn{{}, {}} + transport := &workerCompletionTransport{ + intake: intake, + completions: []grpc.ClientConnInterface{completions[0], completions[1]}, + } + desc := &protos.TaskHubSidecarService_ServiceDesc.Streams[1] + method := protos.TaskHubSidecarService_CompleteOrchestratorTaskWithExtendedSession_FullMethodName + + var cancels []context.CancelFunc + for range 2 * workerSessionStreamsPerConnection { + ctx, cancel := context.WithCancel(context.Background()) + cancels = append(cancels, cancel) + _, err := transport.NewStream(ctx, desc, method) + require.NoError(t, err) + } + require.EqualValues(t, workerSessionStreamsPerConnection, completions[0].opened.Load()) + require.EqualValues(t, workerSessionStreamsPerConnection, completions[1].opened.Load()) + require.Zero(t, intake.opened.Load()) + + _, err := transport.NewStream(context.Background(), desc, protos.TaskHubSidecarService_CompleteEntityTaskWithExtendedSession_FullMethodName) + require.ErrorIs(t, err, errSessionStreamBudgetFull) + + // An ended stream returns its budget. + cancels[0]() + require.Eventually(t, func() bool { + ctx, cancel := context.WithCancel(context.Background()) + cancels = append(cancels, cancel) + _, err := transport.NewStream(ctx, desc, method) + return err == nil + }, time.Second, time.Millisecond) + + // Other streams stay on intake. + _, err = transport.NewStream(context.Background(), desc, protos.TaskHubSidecarService_GetWorkItems_FullMethodName) + require.NoError(t, err) + require.EqualValues(t, 1, intake.opened.Load()) + for _, cancel := range cancels { + cancel() + } + + require.True(t, isWorkerCompletionMethod(protos.TaskHubSidecarService_ReleaseTaskOrchestratorWorkItem_FullMethodName)) + require.True(t, isWorkerCompletionMethod(protos.TaskHubSidecarService_ReleaseTaskEntityWorkItem_FullMethodName)) +} + +func TestBudgetedStreamReturnsBudgetOnRecvError(t *testing.T) { + transport := &workerCompletionTransport{ + intake: &fakeClientConn{}, + completions: []grpc.ClientConnInterface{&erroringStreamConn{}}, + } + stream, err := transport.NewStream(context.Background(), &grpc.StreamDesc{ServerStreams: true}, + protos.TaskHubSidecarService_CompleteOrchestratorTaskWithExtendedSession_FullMethodName) + require.NoError(t, err) + require.Equal(t, []int{1}, transport.sessionStreams) + require.ErrorIs(t, stream.RecvMsg(nil), io.EOF) + require.Equal(t, []int{0}, transport.sessionStreams) +} + +type erroringStreamConn struct{ fakeClientConn } + +func (*erroringStreamConn) NewStream(context.Context, *grpc.StreamDesc, string, ...grpc.CallOption) (grpc.ClientStream, error) { + return erroringClientStream{}, nil +} + +type erroringClientStream struct{ grpc.ClientStream } + +func (erroringClientStream) RecvMsg(any) error { return io.EOF } From e3e04f2dbdeb74ebfd49fd2a96546f909a85050c Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Fri, 6 Nov 2026 16:36:31 -0800 Subject: [PATCH 05/21] Add DTS emulator conformance tests for extended sessions The tests cover the shared scenarios that a black-box SDK test can drive: multi-turn sessions, continue-as-new, a lost acknowledgement, a cut batch, a broken idle stream, the release race, entities, the history limit, and the stream budget. A stream interceptor proves that the session path was used and injects the faults. The tests skip when the endpoint returns Unimplemented, as the current public emulator does. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 1 + .../dts_extended_sessions_test.go | 571 ++++++++++++++++++ 2 files changed, 572 insertions(+) create mode 100644 tests/durabletaskscheduler/dts_extended_sessions_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 9083bcc1..b1676123 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added opt-in extended sessions to gRPC workers with `WithExtendedSessions`. A session keeps an orchestration or entity loaded between turns, so DTS sends only the new messages and the worker skips the full replay. Sessions are off by default. When the service returns `Unimplemented`, the worker completes work items without sessions until it reconnects. - Added required completion-transport isolation to gRPC workers: one intake and three dedicated completion/abandon connections by default, with a configurable 1–8 completion budget and bounded generation-safe retirement. `WithWorkerCompletionConnections(0)` is invalid. Borrowed workers and client listeners require separate caller-owned channels via `WithWorkerCompletionTransports`; owning factories must return distinct connections and non-nil closers for every invocation. - Added `ResourceId` connection-string support for every DTS authentication mode and optional `AuthorityHost` configuration for SDK-created DefaultAzure, WorkloadIdentity, Environment, and InteractiveBrowser credentials. Caller-supplied credentials own their authority; managed identity and developer-tool cloud configuration remain separate. - Added a flat catalogue of directly runnable DTS samples with outcome assertions, owned-resource cleanup, and process-level E2E validation selected by sample name using Go's `-run` flag. diff --git a/tests/durabletaskscheduler/dts_extended_sessions_test.go b/tests/durabletaskscheduler/dts_extended_sessions_test.go new file mode 100644 index 00000000..2810ab84 --- /dev/null +++ b/tests/durabletaskscheduler/dts_extended_sessions_test.go @@ -0,0 +1,571 @@ +package durabletaskscheduler_test + +import ( + "context" + "errors" + "fmt" + "io" + "slices" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/google/uuid" + "github.com/microsoft/durabletask-go/api" + durabletaskclient "github.com/microsoft/durabletask-go/client" + "github.com/microsoft/durabletask-go/durabletaskscheduler" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/microsoft/durabletask-go/task" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// sessionTracker observes extended-session streams and can inject faults. +// The fault switches are one-shot. +type sessionTracker struct { + acks atomic.Int32 + entityAcks atomic.Int32 + open atomic.Int32 + maxOpen atomic.Int32 + unaryOrchestrations atomic.Int32 + releases atomic.Int32 + + dropAck atomic.Bool + cutBatch atomic.Bool + breakWhenIdle atomic.Bool + + mu sync.Mutex + codes map[codes.Code]int +} + +func newSessionTracker() *sessionTracker { + return &sessionTracker{codes: make(map[codes.Code]int)} +} + +func (tr *sessionTracker) recordCode(err error) { + if err == nil || errors.Is(err, io.EOF) { + return + } + tr.mu.Lock() + tr.codes[status.Code(err)]++ + tr.mu.Unlock() +} + +func (tr *sessionTracker) code(code codes.Code) int { + tr.mu.Lock() + defer tr.mu.Unlock() + return tr.codes[code] +} + +func (tr *sessionTracker) streamInterceptor() grpc.StreamClientInterceptor { + return func( + ctx context.Context, + desc *grpc.StreamDesc, + cc *grpc.ClientConn, + method string, + streamer grpc.Streamer, + opts ...grpc.CallOption, + ) (grpc.ClientStream, error) { + stream, err := streamer(ctx, desc, cc, method, opts...) + entity := method == protos.TaskHubSidecarService_CompleteEntityTaskWithExtendedSession_FullMethodName + if err != nil || (!entity && method != protos.TaskHubSidecarService_CompleteOrchestratorTaskWithExtendedSession_FullMethodName) { + tr.recordCode(err) + return stream, err + } + open := tr.open.Add(1) + for { + current := tr.maxOpen.Load() + if open <= current || tr.maxOpen.CompareAndSwap(current, open) { + break + } + } + tracked := &trackedSessionStream{ClientStream: stream, tracker: tr, entity: entity} + context.AfterFunc(ctx, tracked.finish) + return tracked, nil + } +} + +func (tr *sessionTracker) unaryInterceptor() grpc.UnaryClientInterceptor { + return func( + ctx context.Context, + method string, + req, reply any, + cc *grpc.ClientConn, + invoker grpc.UnaryInvoker, + opts ...grpc.CallOption, + ) error { + switch method { + case protos.TaskHubSidecarService_CompleteOrchestratorTask_FullMethodName: + tr.unaryOrchestrations.Add(1) + case protos.TaskHubSidecarService_ReleaseTaskOrchestratorWorkItem_FullMethodName, + protos.TaskHubSidecarService_ReleaseTaskEntityWorkItem_FullMethodName: + tr.releases.Add(1) + } + return invoker(ctx, method, req, reply, cc, opts...) + } +} + +type trackedSessionStream struct { + grpc.ClientStream + tracker *sessionTracker + entity bool + once sync.Once + broken bool +} + +func (s *trackedSessionStream) finish() { + s.once.Do(func() { s.tracker.open.Add(-1) }) +} + +func (s *trackedSessionStream) fail(message string) error { + s.finish() + err := status.Error(codes.Unavailable, message) + s.tracker.recordCode(err) + return err +} + +func (s *trackedSessionStream) RecvMsg(m any) error { + if s.broken { + return s.fail("injected: session stream broke while idle") + } + err := s.ClientStream.RecvMsg(m) + if err != nil { + s.finish() + s.tracker.recordCode(err) + return err + } + response := m.(*protos.CompleteTaskWithExtendedSessionResponse) + switch { + case response.GetAck() != nil: + if s.tracker.dropAck.CompareAndSwap(true, false) { + return s.fail("injected: acknowledgement lost") + } + if s.entity { + s.tracker.entityAcks.Add(1) + } else { + s.tracker.acks.Add(1) + } + s.broken = s.tracker.breakWhenIdle.CompareAndSwap(true, false) + case response.GetNewMessages() != nil: + if s.tracker.cutBatch.CompareAndSwap(true, false) { + return s.fail("injected: batch cut off after the first chunk") + } + } + return nil +} + +func startSessionEmulatorWorker( + t *testing.T, + registry *task.TaskRegistry, + tracker *sessionTracker, + sessions *durabletaskclient.ExtendedSessionOptions, + extra ...durabletaskclient.TaskHubGrpcWorkerOption, +) *durabletaskscheduler.Client { + t.Helper() + options := emulatorOptions(t) + options.StreamInterceptors = append(slices.Clone(options.StreamInterceptors), tracker.streamInterceptor()) + options.UnaryInterceptors = append(slices.Clone(options.UnaryInterceptors), tracker.unaryInterceptor()) + workerOptions := []durabletaskclient.TaskHubGrpcWorkerOption{ + durabletaskclient.WithAutoWorkItemFilters(), + durabletaskclient.WithMaxConcurrentOrchestrationWorkItems(8), + durabletaskclient.WithMaxConcurrentActivityWorkItems(8), + durabletaskclient.WithMaxConcurrentEntityWorkItems(8), + durabletaskclient.WithWorkerSilentDisconnectTimeout(15 * time.Second), + } + if sessions != nil { + workerOptions = append(workerOptions, durabletaskclient.WithExtendedSessions(*sessions)) + } + client, _ := startEmulatorWithOptions(t, options, registry, append(workerOptions, extra...)...) + return client +} + +var ( + sessionProbeOnce sync.Once + sessionProbeSupported bool + sessionProbeErr error +) + +// requireExtendedSessions skips the test when the endpoint does not support +// extended sessions, for example the public emulator before the feature. +func requireExtendedSessions(t *testing.T) { + t.Helper() + emulatorOptions(t) + sessionProbeOnce.Do(func() { + registry := task.NewTaskRegistry() + if err := registry.AddActivityN("SessionProbeActivity", func(task.ActivityContext) (any, error) { return nil, nil }); err != nil { + sessionProbeErr = err + return + } + if err := registry.AddOrchestratorN("SessionProbe", func(ctx *task.OrchestrationContext) (any, error) { + return nil, ctx.CallActivity("SessionProbeActivity").Await(nil) + }); err != nil { + sessionProbeErr = err + return + } + tracker := newSessionTracker() + client := startSessionEmulatorWorker(t, registry, tracker, &durabletaskclient.ExtendedSessionOptions{}) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + id, err := client.ScheduleNewOrchestration(ctx, "SessionProbe") + if err != nil { + sessionProbeErr = err + return + } + if _, err := client.WaitForOrchestrationCompletion(ctx, id); err != nil { + sessionProbeErr = err + return + } + sessionProbeSupported = tracker.acks.Load() > 0 + if !sessionProbeSupported && tracker.code(codes.Unimplemented) == 0 { + sessionProbeErr = fmt.Errorf("the session probe saw neither an acknowledgement nor Unimplemented") + } + }) + require.NoError(t, sessionProbeErr) + if !sessionProbeSupported { + t.Skip("the DTS endpoint returns Unimplemented for extended sessions") + } +} + +func sessionActivityChain(name string, starts *atomic.Int32, steps int) (*task.TaskRegistry, error) { + registry := task.NewTaskRegistry() + if err := registry.AddActivityN(name+"Step", func(ctx task.ActivityContext) (any, error) { + var input int + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + return input + 1, nil + }); err != nil { + return nil, err + } + err := registry.AddOrchestratorN(name, func(ctx *task.OrchestrationContext) (any, error) { + starts.Add(1) + value := 0 + for range steps { + if err := ctx.CallActivity(name+"Step", task.WithActivityInput(value)).Await(&value); err != nil { + return nil, err + } + } + return value, nil + }) + return registry, err +} + +func runSessionOrchestration( + t *testing.T, + client *durabletaskscheduler.Client, + name string, + opts ...api.NewOrchestrationOptions, +) *api.OrchestrationMetadata { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + id, err := client.ScheduleNewOrchestration(ctx, name, opts...) + require.NoError(t, err) + metadata, err := client.WaitForOrchestrationCompletion(ctx, id) + require.NoError(t, err) + return metadata +} + +func requireCompletedValue(t *testing.T, metadata *api.OrchestrationMetadata, want int) { + t.Helper() + require.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, metadata.RuntimeStatus, + "failure: %v", metadata.FailureDetails) + var got int + require.NoError(t, metadata.ReadOutput(&got)) + require.Equal(t, want, got) +} + +// Scenario 1: a multi-turn session, then a terminal unary completion. +func TestDTSExtendedSessionMultiTurn(t *testing.T) { + requireExtendedSessions(t) + starts := new(atomic.Int32) + registry, err := sessionActivityChain("SessionMultiTurn", starts, 4) + require.NoError(t, err) + tracker := newSessionTracker() + client := startSessionEmulatorWorker(t, registry, tracker, &durabletaskclient.ExtendedSessionOptions{}) + + requireCompletedValue(t, runSessionOrchestration(t, client, "SessionMultiTurn"), 4) + require.EqualValues(t, 1, starts.Load(), "the session path must run every turn without a replay") + require.GreaterOrEqual(t, tracker.acks.Load(), int32(4)) + require.GreaterOrEqual(t, tracker.unaryOrchestrations.Load(), int32(1)) +} + +// Scenario 2: a session, continue-as-new, and a terminal completion. +func TestDTSExtendedSessionContinueAsNew(t *testing.T) { + requireExtendedSessions(t) + starts := new(atomic.Int32) + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddActivityN("SessionCANStep", func(task.ActivityContext) (any, error) { return nil, nil })) + require.NoError(t, registry.AddOrchestratorN("SessionCAN", func(ctx *task.OrchestrationContext) (any, error) { + starts.Add(1) + var generation int + if err := ctx.GetInput(&generation); err != nil { + return nil, err + } + for range 2 { + if err := ctx.CallActivity("SessionCANStep").Await(nil); err != nil { + return nil, err + } + } + if generation < 2 { + ctx.ContinueAsNew(generation + 1) + return nil, nil + } + return generation, nil + })) + tracker := newSessionTracker() + client := startSessionEmulatorWorker(t, registry, tracker, &durabletaskclient.ExtendedSessionOptions{}) + + requireCompletedValue(t, runSessionOrchestration(t, client, "SessionCAN", api.WithInput(0)), 2) + require.EqualValues(t, 3, starts.Load(), "each execution must start once and then continue in its session") + require.GreaterOrEqual(t, tracker.acks.Load(), int32(6)) +} + +// Scenarios 3, 4 and 5: lost acknowledgement, a batch cut off after its +// first chunk, and a session stream that breaks while idle. The worker +// recovers with a full replay and the result is the same. +func TestDTSExtendedSessionRecovery(t *testing.T) { + requireExtendedSessions(t) + tests := []struct { + name string + inject func(*sessionTracker) + check func(*testing.T, *sessionTracker) + }{ + { + name: "lost acknowledgement", + inject: func(tr *sessionTracker) { tr.dropAck.Store(true) }, + check: func(t *testing.T, tr *sessionTracker) { + require.False(t, tr.dropAck.Load(), "the fault was not injected") + require.GreaterOrEqual(t, tr.code(codes.NotFound), 1, "a retry with the old count must get NotFound") + }, + }, + { + name: "batch cut off", + inject: func(tr *sessionTracker) { tr.cutBatch.Store(true) }, + check: func(t *testing.T, tr *sessionTracker) { + require.False(t, tr.cutBatch.Load(), "the fault was not injected") + require.GreaterOrEqual(t, tr.releases.Load(), int32(1)) + }, + }, + { + name: "stream breaks while idle", + inject: func(tr *sessionTracker) { tr.breakWhenIdle.Store(true) }, + check: func(t *testing.T, tr *sessionTracker) { + require.False(t, tr.breakWhenIdle.Load(), "the fault was not injected") + require.GreaterOrEqual(t, tr.releases.Load(), int32(1)) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + name := "SessionRecovery" + strings.ReplaceAll(strings.ToUpper(tt.name[:1])+tt.name[1:], " ", "") + starts := new(atomic.Int32) + registry, err := sessionActivityChain(name, starts, 3) + require.NoError(t, err) + tracker := newSessionTracker() + tt.inject(tracker) + client := startSessionEmulatorWorker(t, registry, tracker, &durabletaskclient.ExtendedSessionOptions{}) + requireCompletedValue(t, runSessionOrchestration(t, client, name), 3) + tt.check(t, tracker) + require.Greater(t, starts.Load(), int32(1), "recovery must use a full replay") + }) + } +} + +// Scenario 7: the worker releases idle sessions while new messages arrive. +func TestDTSExtendedSessionReleaseRace(t *testing.T) { + requireExtendedSessions(t) + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("SessionReleaseRace", func(ctx *task.OrchestrationContext) (any, error) { + sum := 0 + for range 8 { + var value int + if err := ctx.WaitForSingleEvent("value", -1).Await(&value); err != nil { + return nil, err + } + sum += value + } + return sum, nil + })) + tracker := newSessionTracker() + client := startSessionEmulatorWorker(t, registry, tracker, + &durabletaskclient.ExtendedSessionOptions{IdleTimeout: 100 * time.Millisecond}) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + id, err := client.ScheduleNewOrchestration(ctx, "SessionReleaseRace") + require.NoError(t, err) + _, err = client.WaitForOrchestrationStart(ctx, id) + require.NoError(t, err) + for i := 1; i <= 8; i++ { + time.Sleep(time.Duration(40*(i%4)) * time.Millisecond) + require.NoError(t, client.RaiseEvent(ctx, id, "value", api.WithEventPayload(i))) + } + metadata, err := client.WaitForOrchestrationCompletion(ctx, id) + require.NoError(t, err) + requireCompletedValue(t, metadata, 36) + require.GreaterOrEqual(t, tracker.releases.Load(), int32(1), "the idle timeout must have released a session") +} + +// Scenario 9: entity operations across turns, a failed batch, and deletion. +func TestDTSExtendedSessionEntities(t *testing.T) { + requireExtendedSessions(t) + var failBatches atomic.Bool + var failedBatches atomic.Int32 + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddEntityFactoryN("SessionCounter", func(task.EntityFactoryContext) (task.EntityBatch, error) { + if failBatches.Load() { + failedBatches.Add(1) + return task.EntityBatch{}, errors.New("injected entity batch failure") + } + return task.EntityBatch{Entity: func(ctx *task.EntityContext) (any, error) { + if ctx.Operation == "delete" { + ctx.DeleteState() + return nil, nil + } + var value int + if ctx.HasState() { + if err := ctx.GetState(&value); err != nil { + return nil, err + } + } + return nil, ctx.SetState(value + 1) + }}, nil + })) + tracker := newSessionTracker() + client := startSessionEmulatorWorker(t, registry, tracker, &durabletaskclient.ExtendedSessionOptions{}) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + entityID := api.NewEntityID("SessionCounter", uuid.NewString()) + requireState := func(want string) { + t.Helper() + require.Eventually(t, func() bool { + metadata, err := client.GetEntity(ctx, entityID) + if err != nil { + return false + } + if want == "" { + return metadata == nil || !metadata.HasState + } + return metadata != nil && metadata.SerializedState == want + }, time.Minute, 50*time.Millisecond, "entity state did not become %q", want) + } + + for range 3 { + require.NoError(t, client.SignalEntity(ctx, entityID, "add")) + time.Sleep(300 * time.Millisecond) + } + requireState("3") + require.GreaterOrEqual(t, tracker.entityAcks.Load(), int32(1), "entity batches must use sessions") + + failBatches.Store(true) + require.NoError(t, client.SignalEntity(ctx, entityID, "add")) + require.Eventually(t, func() bool { return failedBatches.Load() > 0 }, time.Minute, 10*time.Millisecond) + failBatches.Store(false) + requireState("4") + + require.NoError(t, client.SignalEntity(ctx, entityID, "delete")) + requireState("") +} + +// Scenario 10: a session crosses the history limit, and the result equals a +// worker without sessions. +func TestDTSExtendedSessionHistoryLimitMatchesReplay(t *testing.T) { + requireExtendedSessions(t) + run := func(sessions *durabletaskclient.ExtendedSessionOptions) *api.OrchestrationMetadata { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddActivityN("SessionLimitStep", func(task.ActivityContext) (any, error) { return nil, nil })) + require.NoError(t, registry.AddOrchestratorN("SessionHistoryLimit", func(ctx *task.OrchestrationContext) (any, error) { + for { + if err := ctx.CallActivity("SessionLimitStep").Await(nil); err != nil { + return nil, err + } + } + })) + tracker := newSessionTracker() + options := emulatorOptions(t) + options.StreamInterceptors = append(slices.Clone(options.StreamInterceptors), tracker.streamInterceptor()) + workerOptions := []durabletaskclient.TaskHubGrpcWorkerOption{ + durabletaskclient.WithAutoWorkItemFilters(), + durabletaskclient.WithMaxConcurrentOrchestrationWorkItems(4), + durabletaskclient.WithTaskExecutorOptions(task.WithOrchestrationOptions(task.OrchestrationOptions{MaxHistoryEvents: 30})), + } + if sessions != nil { + workerOptions = append(workerOptions, durabletaskclient.WithExtendedSessions(*sessions)) + } + client, worker := startEmulatorWithOptions(t, options, registry, workerOptions...) + metadata := runSessionOrchestration(t, client, "SessionHistoryLimit") + if sessions != nil { + require.Greater(t, tracker.acks.Load(), int32(1), "the first run must use sessions") + } + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + require.NoError(t, worker.Shutdown(shutdownCtx)) + return metadata + } + withSessions := run(&durabletaskclient.ExtendedSessionOptions{}) + withoutSessions := run(nil) + require.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, withSessions.RuntimeStatus) + require.Equal(t, withoutSessions.RuntimeStatus, withSessions.RuntimeStatus) + require.NotNil(t, withSessions.FailureDetails) + require.Equal(t, withoutSessions.FailureDetails.ErrorType, withSessions.FailureDetails.ErrorType) + normalize := func(metadata *api.OrchestrationMetadata) string { + return strings.ReplaceAll(metadata.FailureDetails.ErrorMessage, string(metadata.InstanceID), "") + } + require.Equal(t, normalize(withoutSessions), normalize(withSessions)) +} + +// Scenario 11: when the stream budget is full, turns use the unary RPC, and +// sessions start again once streams are free. +func TestDTSExtendedSessionStreamBudget(t *testing.T) { + requireExtendedSessions(t) + const instances = 56 + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("SessionBudget", func(ctx *task.OrchestrationContext) (any, error) { + var value int + if err := ctx.WaitForSingleEvent("go", -1).Await(&value); err != nil { + return nil, err + } + return value, nil + })) + tracker := newSessionTracker() + client := startSessionEmulatorWorker(t, registry, tracker, &durabletaskclient.ExtendedSessionOptions{}, + durabletaskclient.WithMaxConcurrentOrchestrationWorkItems(2*instances), + durabletaskclient.WithWorkerCompletionConnections(1), + ) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + ids := make([]api.InstanceID, 0, instances) + for range instances { + id, err := client.ScheduleNewOrchestration(ctx, "SessionBudget") + require.NoError(t, err) + ids = append(ids, id) + } + require.Eventually(t, func() bool { + return tracker.acks.Load()+tracker.unaryOrchestrations.Load() >= instances + }, 2*time.Minute, 50*time.Millisecond) + require.LessOrEqual(t, tracker.maxOpen.Load(), int32(50), "one connection must not exceed its session stream budget") + require.Greater(t, tracker.unaryOrchestrations.Load(), int32(0), "turns over the budget must use the unary RPC") + for i, id := range ids { + require.NoError(t, client.RaiseEvent(ctx, id, "go", api.WithEventPayload(i))) + } + for i, id := range ids { + metadata, err := client.WaitForOrchestrationCompletion(ctx, id) + require.NoError(t, err) + requireCompletedValue(t, metadata, i) + } + + // With free streams, a new instance uses a session again. + acksBefore := tracker.acks.Load() + id, err := client.ScheduleNewOrchestration(ctx, "SessionBudget") + require.NoError(t, err) + require.Eventually(t, func() bool { return tracker.acks.Load() > acksBefore }, time.Minute, 20*time.Millisecond) + require.NoError(t, client.RaiseEvent(ctx, id, "go", api.WithEventPayload(7))) + metadata, err := client.WaitForOrchestrationCompletion(ctx, id) + require.NoError(t, err) + requireCompletedValue(t, metadata, 7) +} From c7c4adea17ede3a0312c0d6ec0c3bd4e2dd6b127 Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Fri, 6 Nov 2026 16:37:36 -0800 Subject: [PATCH 06/21] Bound abandons in later session turns by the cleanup deadline After the first turn, a work item belongs to a session. Its abandon now uses the same deadline-bounded cleanup as release, and a failed cleanup reconnects intake. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- client/grpc_worker_processor.go | 40 ++++++++++++++++++------------ client/grpc_worker_session_test.go | 21 ++++++++++++++++ 2 files changed, 45 insertions(+), 16 deletions(-) diff --git a/client/grpc_worker_processor.go b/client/grpc_worker_processor.go index 8797781e..e8b5134b 100644 --- a/client/grpc_worker_processor.go +++ b/client/grpc_worker_processor.go @@ -321,7 +321,9 @@ func (w *TaskHubGrpcWorker) processOrchestrationWorkItem( traceContext: request.OrchestrationTraceContext, parentTrace: executionStartedTraceContext(pastEvents, request.NewEvents), } - response, ok := w.orchestratorResponse(ctx, client, turn, results, err) + response, ok := w.orchestratorResponse(ctx, turn, results, err, func() { + w.abandonOrchestration(ctx, client, completionToken) + }) if !ok { if session != nil { session.Close() @@ -345,13 +347,13 @@ type orchestrationTurn struct { } // orchestratorResponse builds the response for one turn. It returns false -// after it abandoned the work item. +// after it called abandon. func (w *TaskHubGrpcWorker) orchestratorResponse( ctx context.Context, - client protos.TaskHubSidecarServiceClient, turn orchestrationTurn, results *task.ExecutionResults, err error, + abandon func(), ) (*protos.OrchestratorResponse, bool) { response := &protos.OrchestratorResponse{ InstanceId: turn.instanceID, @@ -392,7 +394,7 @@ func (w *TaskHubGrpcWorker) orchestratorResponse( } if err := largepayload.TransformOrchestratorResponse(ctx, w.options.largePayloads, response); err != nil { w.logger.Errorf("%s: failed to externalize orchestration response payloads: %v", turn.instanceID, err) - w.abandonOrchestration(ctx, client, turn.token) + abandon() return nil, false } return failOversizedOrchestratorResponse(response, w.options.maxOrchestratorCompletionBytes), true @@ -492,7 +494,9 @@ func (w *TaskHubGrpcWorker) runOrchestrationSession( } session = next var ok bool - if response, ok = w.orchestratorResponse(ctx, client, turn, results, nil); !ok { + if response, ok = w.orchestratorResponse(ctx, turn, results, nil, func() { + w.cleanupSessionWorkItem(ctx, scope, turn.instanceID, "abandon", abandonOrchestrationOnce(client, turn.token)) + }); !ok { if session != nil { session.Close() } @@ -804,7 +808,9 @@ func (w *TaskHubGrpcWorker) processEntityWorkItem( completionToken string, request *protos.EntityRequest, ) { - result, ok := w.entityBatchResult(ctx, client, completionToken, request) + result, ok := w.entityBatchResult(ctx, completionToken, request, func() { + w.abandonEntity(ctx, client, completionToken) + }) if !ok { return } @@ -816,34 +822,34 @@ func (w *TaskHubGrpcWorker) processEntityWorkItem( } // entityBatchResult runs one entity batch and returns the result to send. -// It returns false after it abandoned the work item. +// It returns false after it called abandon. func (w *TaskHubGrpcWorker) entityBatchResult( ctx context.Context, - client protos.TaskHubSidecarServiceClient, completionToken string, request *protos.EntityRequest, + abandon func(), ) (*protos.EntityBatchResult, bool) { batch, operationInfos, err := entityBatchFromRequestV2(request) if err != nil { w.logger.Errorf("invalid V2 entity work item: %v", err) - w.abandonEntity(ctx, client, completionToken) + abandon() return nil, false } entityID, parseErr := api.EntityIDFromString(batch.GetInstanceId()) if parseErr != nil { w.logger.Errorf("%s: invalid entity instance ID: %v", batch.GetInstanceId(), parseErr) - w.abandonEntity(ctx, client, completionToken) + abandon() return nil, false } if !matchesEntityWorkItemFilters(w.options.workItemFilters, entityID.Name) { w.logger.Warnf("%s: entity work item does not match configured filters; abandoning it", request.GetInstanceId()) - w.abandonEntity(ctx, client, completionToken) + abandon() return nil, false } if err := largepayload.TransformEntityBatchRequest(ctx, w.options.largePayloads, batch); err != nil { w.logger.Errorf("%s: failed to hydrate entity batch payloads: %v", request.GetInstanceId(), err) if entityProcessingCanceled(ctx, err) { - w.abandonEntity(ctx, client, completionToken) + abandon() return nil, false } return newEntityBatchFailure(completionToken, err), true @@ -851,14 +857,14 @@ func (w *TaskHubGrpcWorker) entityBatchResult( executor, ok := w.executor.(task.EntityExecutor) if !ok { w.logger.Error("task executor does not support entity work items") - w.abandonEntity(ctx, client, completionToken) + abandon() return nil, false } result, err := executeEntitySafely(ctx, executor, batch) if err != nil { if entityProcessingCanceled(ctx, err) { w.logger.Warnf("%s: entity execution canceled; abandoning work item", request.GetInstanceId()) - w.abandonEntity(ctx, client, completionToken) + abandon() return nil, false } w.logger.Errorf("%s: entity execution failed: %v", request.GetInstanceId(), err) @@ -876,7 +882,7 @@ func (w *TaskHubGrpcWorker) entityBatchResult( if err := largepayload.TransformEntityBatchResult(ctx, w.options.largePayloads, result); err != nil { w.logger.Errorf("%s: failed to externalize entity batch payloads: %v", request.GetInstanceId(), err) if entityProcessingCanceled(ctx, err) { - w.abandonEntity(ctx, client, completionToken) + abandon() return nil, false } return newEntityBatchFailure(completionToken, err), true @@ -931,10 +937,12 @@ func (w *TaskHubGrpcWorker) runEntitySession( return } var ok bool - result, ok = w.entityBatchResult(ctx, client, completionToken, &protos.EntityRequest{ + result, ok = w.entityBatchResult(ctx, completionToken, &protos.EntityRequest{ InstanceId: instanceID, EntityState: state, OperationRequests: entityOperationEvents(batch), + }, func() { + w.cleanupSessionWorkItem(ctx, scope, instanceID, "abandon", abandonEntityOnce(client, completionToken)) }) if !ok { return diff --git a/client/grpc_worker_session_test.go b/client/grpc_worker_session_test.go index c74d44f1..4c036b35 100644 --- a/client/grpc_worker_session_test.go +++ b/client/grpc_worker_session_test.go @@ -831,6 +831,27 @@ func TestWorkerEntitySessionUsesCachedState(t *testing.T) { require.Len(t, requests[1].Operations, 2, "control events are not entity operations") } +func TestWorkerEntitySessionAbandonsInvalidBatch(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddEntityN("counter", func(*task.EntityContext) (any, error) { return nil, nil })) + client := newSessionSchedulerClient() + startSessionWorker(t, registry, client) + client.stream.results <- fakeWorkItemResult{item: &protos.WorkItem{ + Request: &protos.WorkItem_EntityRequestV2{EntityRequestV2: entitySessionRequest("@counter@b", entitySignal("noop"))}, + CompletionToken: "entity-token", + }} + first := client.nextCall(t) + first.stream.ack(0) + first.stream.batch(helpers.NewTaskCompletedEvent(1, nil)) + require.Eventually(t, func() bool { + client.mu.Lock() + defer client.mu.Unlock() + return client.entityAbandonAttempts == 1 + }, 5*time.Second, time.Millisecond) + require.Empty(t, client.unaryEntities()) + require.Zero(t, client.releases.Load()) +} + func TestWithExtendedSessionsValidation(t *testing.T) { newWorker := func(opts ...TaskHubGrpcWorkerOption) error { _, err := newTaskHubGrpcWorker( From 686e2f44f76292c398f4faa2a13b0462611807af Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Fri, 6 Nov 2026 16:49:54 -0800 Subject: [PATCH 07/21] Fix extended-session review findings in the worker - Copy ExtendedSessionOptions per worker. Workers built from one option value shared one struct, so resolved limits leaked between them. - After an acknowledged session turn, a failed unary completion abandons with the deadline-bounded cleanup, so a stuck abandon reconnects intake. - Remove MaxCachedBytes and LRU eviction. The estimate counted history that the executor already drops, and the session count limits bound memory. - Merge the two entity completion helpers into one with a count parameter. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- client/grpc_worker_processor.go | 59 +++++-------- client/grpc_worker_session.go | 131 +++++++++-------------------- client/grpc_worker_session_test.go | 119 ++++++++++++++++---------- 3 files changed, 137 insertions(+), 172 deletions(-) diff --git a/client/grpc_worker_processor.go b/client/grpc_worker_processor.go index e8b5134b..fb704b6e 100644 --- a/client/grpc_worker_processor.go +++ b/client/grpc_worker_processor.go @@ -331,10 +331,10 @@ func (w *TaskHubGrpcWorker) processOrchestrationWorkItem( return } if session != nil { - w.runOrchestrationSession(ctx, client, scope, turn, session, response, historySize(pastEvents, request.NewEvents)) + w.runOrchestrationSession(ctx, client, scope, turn, session, response) return } - w.completeOrchestration(ctx, client, turn, response, 0) + w.completeOrchestration(ctx, client, nil, turn, response, 0) } // orchestrationTurn holds what every turn of one orchestration work item @@ -402,10 +402,11 @@ func (w *TaskHubGrpcWorker) orchestratorResponse( // completeOrchestration sends a turn with the unary RPC. The count and token // are set right before sending, because earlier steps may rebuild the -// response. +// response. scope is set for a work item that is in a session. func (w *TaskHubGrpcWorker) completeOrchestration( ctx context.Context, client protos.TaskHubSidecarServiceClient, + scope *workerSessionScope, turn orchestrationTurn, response *protos.OrchestratorResponse, completionCount int32, @@ -422,6 +423,10 @@ func (w *TaskHubGrpcWorker) completeOrchestration( return } w.logger.Errorf("%s: failed to complete orchestration work item: %v", turn.instanceID, err) + if scope != nil { + w.cleanupSessionWorkItem(ctx, scope, turn.instanceID, "abandon", abandonOrchestrationOnce(client, turn.token)) + return + } w.abandonOrchestration(ctx, client, turn.token) } } @@ -436,7 +441,6 @@ func (w *TaskHubGrpcWorker) runOrchestrationSession( turn orchestrationTurn, session task.OrchestratorSession, response *protos.OrchestratorResponse, - sessionBytes int64, ) { lease := scope.run.sessions.acquire(orchestrationSessionKind) defer lease.release() @@ -447,22 +451,21 @@ func (w *TaskHubGrpcWorker) runOrchestrationSession( session = nil } if session == nil { - w.completeOrchestration(ctx, client, turn, response, turnCount) + w.completeOrchestration(ctx, client, sessionScopeAfterAck(scope, turnCount), turn, response, turnCount) return } var historyBase int32 - outcome, batch := w.awaitSessionBatch(ctx, scope, lease, sessionCall{ + outcome, batch := w.awaitSessionBatch(ctx, scope, sessionCall{ instanceID: turn.instanceID, open: func(callCtx context.Context) (grpc.ServerStreamingClient[protos.CompleteTaskWithExtendedSessionResponse], error) { response.CompletionToken = turn.token response.CompletionCount = turnCount return client.CompleteOrchestratorTaskWithExtendedSession(callCtx, response) }, - onAck: func(ack *protos.ExtendedSessionAck) int64 { + onAck: func(ack *protos.ExtendedSessionAck) { session.Commit(response.Actions) historyBase = ack.GetHistoryEventCount() turnCount++ - return sessionBytes }, release: releaseOrchestrationOnce(client, turn.token), abandon: abandonOrchestrationOnce(client, turn.token), @@ -484,7 +487,6 @@ func (w *TaskHubGrpcWorker) runOrchestrationSession( w.cleanupSessionWorkItem(ctx, scope, turn.instanceID, "release", releaseOrchestrationOnce(client, turn.token)) return } - sessionBytes += historySize(nil, batch) results, next, err := session.Resume(ctx, batch, int(historyBase)) if err != nil { w.logger.Errorf("%s: failed to run session batch: %v", turn.instanceID, err) @@ -520,16 +522,6 @@ func canContinueOrchestrationSession(response *protos.OrchestratorResponse) bool return true } -func historySize(eventLists ...[]*protos.HistoryEvent) int64 { - var size int64 - for _, events := range eventLists { - for _, event := range events { - size += int64(proto.Size(event)) - } - } - return size -} - // failOversizedOrchestratorResponse converts a response that modern DTS cannot // accept into a small, non-retriable orchestration failure. The Go worker does // not use the deprecated legacy response-chunking fields; validation against the @@ -818,7 +810,7 @@ func (w *TaskHubGrpcWorker) processEntityWorkItem( w.runEntitySession(ctx, client, scope, request.GetInstanceId(), completionToken, result) return } - w.completeEntityBatchResult(ctx, client, request.GetInstanceId(), result) + w.completeEntityBatchResult(ctx, client, nil, request.GetInstanceId(), result, 0) } // entityBatchResult runs one entity batch and returns the result to send. @@ -911,20 +903,19 @@ func (w *TaskHubGrpcWorker) runEntitySession( ) for { if !streaming || !scope.available() || result.FailureDetails != nil { - w.completeEntityBatchResultWithCount(ctx, client, instanceID, result, turnCount) + w.completeEntityBatchResult(ctx, client, sessionScopeAfterAck(scope, turnCount), instanceID, result, turnCount) return } - outcome, batch := w.awaitSessionBatch(ctx, scope, lease, sessionCall{ + outcome, batch := w.awaitSessionBatch(ctx, scope, sessionCall{ instanceID: instanceID, open: func(callCtx context.Context) (grpc.ServerStreamingClient[protos.CompleteTaskWithExtendedSessionResponse], error) { result.CompletionToken = completionToken result.CompletionCount = turnCount return client.CompleteEntityTaskWithExtendedSession(callCtx, result) }, - onAck: func(*protos.ExtendedSessionAck) int64 { + onAck: func(*protos.ExtendedSessionAck) { state = result.EntityState turnCount++ - return int64(len(state.GetValue())) }, release: releaseEntityOnce(client, completionToken), abandon: abandonEntityOnce(client, completionToken), @@ -1033,21 +1024,13 @@ func entityProcessingCanceled(ctx context.Context, err error) bool { errors.Is(err, context.DeadlineExceeded) } +// completeEntityBatchResult sends an entity result with the unary RPC. The +// count is set right before sending, because a failure result is a new +// message. scope is set for a work item that is in a session. func (w *TaskHubGrpcWorker) completeEntityBatchResult( ctx context.Context, client protos.TaskHubSidecarServiceClient, - instanceID string, - result *protos.EntityBatchResult, -) { - w.completeEntityBatchResultWithCount(ctx, client, instanceID, result, 0) -} - -// completeEntityBatchResultWithCount sends an entity result with the unary -// RPC. The count is set right before sending, because a failure result is a -// new message. -func (w *TaskHubGrpcWorker) completeEntityBatchResultWithCount( - ctx context.Context, - client protos.TaskHubSidecarServiceClient, + scope *workerSessionScope, instanceID string, result *protos.EntityBatchResult, completionCount int32, @@ -1063,6 +1046,10 @@ func (w *TaskHubGrpcWorker) completeEntityBatchResultWithCount( return } w.logger.Errorf("%s: failed to complete entity work item: %v", instanceID, err) + if scope != nil { + w.cleanupSessionWorkItem(ctx, scope, instanceID, "abandon", abandonEntityOnce(client, result.CompletionToken)) + return + } w.abandonEntity(ctx, client, result.CompletionToken) } } diff --git a/client/grpc_worker_session.go b/client/grpc_worker_session.go index 3b44c9a8..b5ef144b 100644 --- a/client/grpc_worker_session.go +++ b/client/grpc_worker_session.go @@ -1,7 +1,6 @@ package client import ( - "container/list" "context" "errors" "fmt" @@ -49,10 +48,6 @@ type ExtendedSessionOptions struct { MaxOrchestrationSessions int // MaxEntitySessions limits waiting entity sessions in the same way. MaxEntitySessions int - // MaxCachedBytes is an estimated memory budget for loaded sessions. When - // it is exceeded, the least recently used waiting sessions are ended. - // Zero means no byte limit. - MaxCachedBytes int64 } // WithExtendedSessions turns on extended sessions. Sessions are off unless @@ -66,16 +61,15 @@ func WithExtendedSessions(options ExtendedSessionOptions) TaskHubGrpcWorkerOptio if options.MaxOrchestrationSessions < 0 || options.MaxEntitySessions < 0 { return fmt.Errorf("extended session limits cannot be negative") } - if options.MaxCachedBytes < 0 { - return fmt.Errorf("extended session cached bytes cannot be negative") + // Each worker resolves its own defaults, so it needs its own copy. + copied := options + if copied.IdleTimeout == 0 { + copied.IdleTimeout = DefaultExtendedSessionTimeout } - if options.IdleTimeout == 0 { - options.IdleTimeout = DefaultExtendedSessionTimeout + if copied.TransferTimeout == 0 { + copied.TransferTimeout = DefaultExtendedSessionTimeout } - if options.TransferTimeout == 0 { - options.TransferTimeout = DefaultExtendedSessionTimeout - } - target.extendedSessions = &options + target.extendedSessions = &copied return nil } } @@ -117,73 +111,36 @@ type sessionManager struct { mu sync.Mutex orchestrations int entities int - bytes int64 - idle *list.List } -// sessionLease is one session's share of the limits. Only the goroutine that -// owns the session uses it, except for eviction. +// sessionLease is one session's place within the limits. type sessionLease struct { manager *sessionManager kind sessionKind - bytes int64 - idle *list.Element - evict chan struct{} - evicted bool - done bool + once sync.Once } func newSessionManager(options *ExtendedSessionOptions) *sessionManager { - return &sessionManager{options: options, idle: list.New()} + return &sessionManager{options: options} } +// acquire returns nil when the limit for kind is reached. func (m *sessionManager) acquire(kind sessionKind) *sessionLease { m.mu.Lock() defer m.mu.Unlock() - count, limit := &m.orchestrations, m.options.MaxOrchestrationSessions - if kind == entitySessionKind { - count, limit = &m.entities, m.options.MaxEntitySessions - } + count, limit := m.counter(kind) if *count >= limit { return nil } *count++ - return &sessionLease{manager: m, kind: kind, evict: make(chan struct{})} -} - -// setIdle marks the session as waiting with the given estimated size. It -// ends the least recently used waiting sessions while over the byte budget. -func (l *sessionLease) setIdle(bytes int64) { - m := l.manager - m.mu.Lock() - defer m.mu.Unlock() - if l.evicted { - return - } - m.bytes += bytes - l.bytes - l.bytes = bytes - if l.idle == nil { - l.idle = m.idle.PushBack(l) - } - for m.options.MaxCachedBytes > 0 && m.bytes > m.options.MaxCachedBytes && m.idle.Len() > 0 { - victim := m.idle.Remove(m.idle.Front()).(*sessionLease) - victim.idle = nil - victim.evicted = true - m.bytes -= victim.bytes - victim.bytes = 0 - close(victim.evict) - } + return &sessionLease{manager: m, kind: kind} } -// setBusy marks the session as running, so it cannot be evicted. -func (l *sessionLease) setBusy() { - m := l.manager - m.mu.Lock() - defer m.mu.Unlock() - if l.idle != nil { - m.idle.Remove(l.idle) - l.idle = nil +func (m *sessionManager) counter(kind sessionKind) (*int, int) { + if kind == entitySessionKind { + return &m.entities, m.options.MaxEntitySessions } + return &m.orchestrations, m.options.MaxOrchestrationSessions } // release returns the lease. It is safe to call on a nil lease. @@ -191,24 +148,12 @@ func (l *sessionLease) release() { if l == nil { return } - m := l.manager - m.mu.Lock() - defer m.mu.Unlock() - if l.done { - return - } - l.done = true - if l.idle != nil { - m.idle.Remove(l.idle) - l.idle = nil - } - m.bytes -= l.bytes - l.bytes = 0 - if l.kind == entitySessionKind { - m.entities-- - } else { - m.orchestrations-- - } + l.once.Do(func() { + l.manager.mu.Lock() + defer l.manager.mu.Unlock() + count, _ := l.manager.counter(l.kind) + *count-- + }) } // workerSessionScope is the intake generation that dispatched a work item. @@ -219,6 +164,16 @@ type workerSessionScope struct { connection *grpcWorkerConnection } +// sessionScopeAfterAck returns scope once the service has accepted at least +// one session turn, so the work item is in a session. Otherwise it returns +// nil, and the work item is handled like any other. +func sessionScopeAfterAck(scope *workerSessionScope, turnCount int32) *workerSessionScope { + if turnCount == 0 { + return nil + } + return scope +} + // available reports whether a new session turn may use the streaming RPC. func (s *workerSessionScope) available() bool { return s != nil && s.run.sessions != nil && @@ -252,9 +207,8 @@ const ( type sessionCall struct { instanceID string open func(context.Context) (grpc.ServerStreamingClient[protos.CompleteTaskWithExtendedSessionResponse], error) - // onAck runs when the service accepts the turn. It returns the estimated - // size of the loaded session. - onAck func(*protos.ExtendedSessionAck) int64 + // onAck runs when the service accepts the turn. + onAck func(*protos.ExtendedSessionAck) release func(context.Context) error abandon func(context.Context) error } @@ -271,11 +225,10 @@ type sessionMessage struct { func (w *TaskHubGrpcWorker) awaitSessionBatch( ctx context.Context, scope *workerSessionScope, - lease *sessionLease, call sessionCall, ) (sessionOutcome, []*protos.HistoryEvent) { for attempt := 1; ; attempt++ { - outcome, batch, preAckErr := w.runSessionStream(ctx, scope, lease, call) + outcome, batch, preAckErr := w.runSessionStream(ctx, scope, call) if preAckErr == nil { return outcome, batch } @@ -307,7 +260,6 @@ func (w *TaskHubGrpcWorker) awaitSessionBatch( func (w *TaskHubGrpcWorker) runSessionStream( ctx context.Context, scope *workerSessionScope, - lease *sessionLease, call sessionCall, ) (sessionOutcome, []*protos.HistoryEvent, error) { streamCtx, cancel := context.WithCancel(ctx) @@ -355,10 +307,10 @@ func (w *TaskHubGrpcWorker) runSessionStream( acked, transferring := false, false var batch []*protos.HistoryEvent for { - // Shutdown and eviction end a session only while it waits. - var stopping, evicted <-chan struct{} + // Shutdown ends a session only while it waits. + var stopping <-chan struct{} if acked && !transferring { - stopping, evicted = scope.run.intakeCtx.Done(), lease.evict + stopping = scope.run.intakeCtx.Done() } select { case <-ctx.Done(): @@ -367,8 +319,6 @@ func (w *TaskHubGrpcWorker) runSessionStream( return end("the work item stream ended", false) case <-stopping: return end("the worker is shutting down", true) - case <-evicted: - return end("the session cache is full", true) case <-timer: switch { case !acked: @@ -399,7 +349,7 @@ func (w *TaskHubGrpcWorker) runSessionStream( return end("the service sent a second acknowledgement", true) } acked = true - lease.setIdle(call.onAck(response.Ack)) + call.onAck(response.Ack) restartTimer(sessionIdleTimer, w.options.extendedSessions.IdleTimeout) case *protos.CompleteTaskWithExtendedSessionResponse_HealthPing: // A ping shows only that the connection is alive. @@ -409,7 +359,6 @@ func (w *TaskHubGrpcWorker) runSessionStream( } if !transferring { transferring = true - lease.setBusy() restartTimer(sessionTransferTimer, w.options.extendedSessions.TransferTimeout) } batch = append(batch, response.NewMessages.GetEvents()...) diff --git a/client/grpc_worker_session_test.go b/client/grpc_worker_session_test.go index 4c036b35..ba569530 100644 --- a/client/grpc_worker_session_test.go +++ b/client/grpc_worker_session_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "io" + "math" "strings" "sync" "sync/atomic" @@ -87,6 +88,7 @@ type sessionSchedulerClient struct { *fakeSchedulerClient calls chan *fakeSessionCall openErr func() error + unaryErr func() error releaseErr func() error abandonErr func() error @@ -111,6 +113,19 @@ func (c *sessionSchedulerClient) GetWorkItems( return c.fakeSchedulerClient.GetWorkItems(ctx, request, opts...) } +func (c *sessionSchedulerClient) CompleteOrchestratorTask( + ctx context.Context, + response *protos.OrchestratorResponse, + opts ...grpc.CallOption, +) (*protos.CompleteTaskResponse, error) { + if c.unaryErr != nil { + if err := c.unaryErr(); err != nil { + return nil, err + } + } + return c.fakeSchedulerClient.CompleteOrchestratorTask(ctx, response, opts...) +} + func (c *sessionSchedulerClient) open(ctx context.Context, call *fakeSessionCall) (grpc.ServerStreamingClient[protos.CompleteTaskWithExtendedSessionResponse], error) { if c.openErr != nil { if err := c.openErr(); err != nil { @@ -688,6 +703,52 @@ func TestWorkerSessionAbandonFailureReconnectsIntake(t *testing.T) { require.Greater(t, h.client.orchestrationAbort.Load(), int32(1)) } +func TestWorkerSessionUnaryFailureAfterAckReconnectsIntake(t *testing.T) { + h := newSessionHarness(t, 1) + h.worker.options.sessionCleanupTimeout = 20 * time.Millisecond + h.client.unaryErr = func() error { return status.Error(codes.InvalidArgument, "rejected") } + h.client.abandonErr = func() error { return status.Error(codes.Unavailable, "down") } + h.dispatch("unary-fails") + first := h.client.nextCall(t) + first.stream.ack(4) + first.stream.batch(taskCompleted(0, "a")...) + require.Eventually(t, func() bool { return h.client.connects.Load() >= 2 }, 5*time.Second, time.Millisecond, + "a failed abandon of a session work item must reconnect intake") + require.Greater(t, h.client.orchestrationAbort.Load(), int32(1)) +} + +func TestWorkerEntitySessionUnaryFailureAfterAckReconnectsIntake(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddEntityN("counter", func(*task.EntityContext) (any, error) { return nil, nil })) + client := newSessionSchedulerClient() + client.entityCompletionErr = status.Error(codes.InvalidArgument, "rejected") + client.entityAbandonFailures = math.MaxInt32 + h := startSessionWorker(t, registry, client) + h.worker.options.sessionCleanupTimeout = 20 * time.Millisecond + // A batch that fails ends the session with the unary RPC. + h.worker.executor = &recordingExecutor{executeEntity: func(_ context.Context, request *protos.EntityBatchRequest) (*protos.EntityBatchResult, error) { + if request.EntityState != nil { + return nil, errors.New("framework failure") + } + return &protos.EntityBatchResult{ + Results: []*protos.OperationResult{{ResultType: &protos.OperationResult_Success{Success: &protos.OperationResultSuccess{}}}}, + EntityState: wrapperspb.String("1"), + }, nil + }} + client.stream.results <- fakeWorkItemResult{item: &protos.WorkItem{ + Request: &protos.WorkItem_EntityRequestV2{EntityRequestV2: entitySessionRequest("@counter@c", entitySignal("add"))}, + CompletionToken: "entity-token", + }} + first := client.nextCall(t) + first.stream.ack(0) + first.stream.batch(entitySignal("add")) + require.Eventually(t, func() bool { return client.connects.Load() >= 2 }, 5*time.Second, time.Millisecond, + "a failed abandon of a session work item must reconnect intake") + client.mu.Lock() + defer client.mu.Unlock() + require.Greater(t, client.entityAbandonAttempts, 1) +} + func TestWorkerSessionClosesWithRetiredGeneration(t *testing.T) { h := newSessionHarness(t, 2) h.dispatch("retired") @@ -722,50 +783,6 @@ func TestWorkerSessionShutdownReleasesIdleSessions(t *testing.T) { require.EqualValues(t, 1, h.client.releases.Load()) } -func TestWorkerSessionEvictsLeastRecentlyUsedIdleSession(t *testing.T) { - manager := newSessionManager(&ExtendedSessionOptions{ - MaxOrchestrationSessions: 3, - MaxCachedBytes: 100, - }) - first := manager.acquire(orchestrationSessionKind) - second := manager.acquire(orchestrationSessionKind) - third := manager.acquire(orchestrationSessionKind) - require.Nil(t, manager.acquire(orchestrationSessionKind)) - first.setIdle(40) - second.setIdle(40) - third.setBusy() - require.Equal(t, int64(80), manager.bytes) - // Touching first makes second the least recently used session. - first.setBusy() - first.setIdle(40) - third.setIdle(40) - select { - case <-second.evict: - default: - t.Fatal("the least recently used idle session must be evicted") - } - select { - case <-first.evict: - t.Fatal("only one session should be evicted") - default: - } - require.Equal(t, int64(80), manager.bytes) - second.release() - second.release() - first.release() - third.release() - require.Zero(t, manager.bytes) - require.Zero(t, manager.orchestrations) -} - -func TestWorkerSessionEvictionReleasesWorkItem(t *testing.T) { - h := newSessionHarness(t, 2, WithExtendedSessions(ExtendedSessionOptions{MaxCachedBytes: 1})) - h.dispatch("evicted") - first := h.client.nextCall(t) - first.stream.ack(4) - require.Eventually(t, func() bool { return h.client.releases.Load() == 1 }, 5*time.Second, time.Millisecond) -} - func entitySessionRequest(instanceID string, operations ...*protos.HistoryEvent) *protos.EntityRequest { return &protos.EntityRequest{InstanceId: instanceID, OperationRequests: operations} } @@ -864,12 +881,24 @@ func TestWithExtendedSessionsValidation(t *testing.T) { } require.Error(t, newWorker(WithExtendedSessions(ExtendedSessionOptions{IdleTimeout: -1}))) require.Error(t, newWorker(WithExtendedSessions(ExtendedSessionOptions{MaxEntitySessions: -1}))) - require.Error(t, newWorker(WithExtendedSessions(ExtendedSessionOptions{MaxCachedBytes: -1}))) require.ErrorContains(t, newWorker( WithMaxConcurrentOrchestrationWorkItems(2), WithExtendedSessions(ExtendedSessionOptions{MaxOrchestrationSessions: 2}), ), "must be lower") + shared := WithExtendedSessions(ExtendedSessionOptions{}) + small, err := newTaskHubGrpcWorker( + func(context.Context) (protos.TaskHubSidecarServiceClient, io.Closer, error) { return nil, nil, nil }, + task.NewTaskRegistry(), api.DefaultLogger(), WithMaxConcurrentOrchestrationWorkItems(4), shared) + require.NoError(t, err) + large, err := newTaskHubGrpcWorker( + func(context.Context) (protos.TaskHubSidecarServiceClient, io.Closer, error) { return nil, nil, nil }, + task.NewTaskRegistry(), api.DefaultLogger(), WithMaxConcurrentOrchestrationWorkItems(40), shared) + require.NoError(t, err) + require.Equal(t, 2, small.options.extendedSessions.MaxOrchestrationSessions, + "workers built from one option value must not share their session options") + require.Equal(t, 20, large.options.extendedSessions.MaxOrchestrationSessions) + options := defaultTaskHubGrpcWorkerOptions() require.NoError(t, WithExtendedSessions(ExtendedSessionOptions{})(&options)) options.maxConcurrentOrchestrations = 1 From d73a16a4442fe08dd0da1d7e82d49d0f5285794d Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Fri, 6 Nov 2026 16:51:25 -0800 Subject: [PATCH 08/21] Deliver the first chunk before cutting the E2E session batch The cut-batch scenario replaced the first chunk with an error, so the worker never saw partial data. It now receives the first chunk, and the stream breaks before the batch ends. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/durabletaskscheduler/dts_extended_sessions_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/durabletaskscheduler/dts_extended_sessions_test.go b/tests/durabletaskscheduler/dts_extended_sessions_test.go index 2810ab84..8f3d323b 100644 --- a/tests/durabletaskscheduler/dts_extended_sessions_test.go +++ b/tests/durabletaskscheduler/dts_extended_sessions_test.go @@ -130,7 +130,7 @@ func (s *trackedSessionStream) fail(message string) error { func (s *trackedSessionStream) RecvMsg(m any) error { if s.broken { - return s.fail("injected: session stream broke while idle") + return s.fail("injected: session stream broke") } err := s.ClientStream.RecvMsg(m) if err != nil { @@ -151,8 +151,9 @@ func (s *trackedSessionStream) RecvMsg(m any) error { } s.broken = s.tracker.breakWhenIdle.CompareAndSwap(true, false) case response.GetNewMessages() != nil: + // Deliver the first chunk, then break the stream before the batch ends. if s.tracker.cutBatch.CompareAndSwap(true, false) { - return s.fail("injected: batch cut off after the first chunk") + s.broken = true } } return nil @@ -344,7 +345,7 @@ func TestDTSExtendedSessionRecovery(t *testing.T) { }, }, { - name: "batch cut off", + name: "batch cut off after the first chunk", inject: func(tr *sessionTracker) { tr.cutBatch.Store(true) }, check: func(t *testing.T, tr *sessionTracker) { require.False(t, tr.cutBatch.Load(), "the fault was not injected") From 3b2cb8f9c23791fa1e7149dda43de6373d021e83 Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Fri, 6 Nov 2026 18:08:01 -0800 Subject: [PATCH 09/21] Fail session E2E tests when the session probe stops early A failed require inside the probe left it unsupported without an error, so every later session test was skipped instead of failing. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/durabletaskscheduler/dts_extended_sessions_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/durabletaskscheduler/dts_extended_sessions_test.go b/tests/durabletaskscheduler/dts_extended_sessions_test.go index 8f3d323b..f264b9c4 100644 --- a/tests/durabletaskscheduler/dts_extended_sessions_test.go +++ b/tests/durabletaskscheduler/dts_extended_sessions_test.go @@ -196,6 +196,9 @@ func requireExtendedSessions(t *testing.T) { t.Helper() emulatorOptions(t) sessionProbeOnce.Do(func() { + // A probe that stops early, for example on a failed require, must fail + // the tests instead of skipping them. + sessionProbeErr = errors.New("the extended-session probe did not finish") registry := task.NewTaskRegistry() if err := registry.AddActivityN("SessionProbeActivity", func(task.ActivityContext) (any, error) { return nil, nil }); err != nil { sessionProbeErr = err @@ -220,6 +223,7 @@ func requireExtendedSessions(t *testing.T) { sessionProbeErr = err return } + sessionProbeErr = nil sessionProbeSupported = tracker.acks.Load() > 0 if !sessionProbeSupported && tracker.code(codes.Unimplemented) == 0 { sessionProbeErr = fmt.Errorf("the session probe saw neither an acknowledgement nor Unimplemented") From ce92a2d556198589e23bbad0a96b7d9eec33257b Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Fri, 6 Nov 2026 18:23:10 -0800 Subject: [PATCH 10/21] Add an extended-session stress and performance harness The tests run only with the stress build tag. They compare workloads with sessions off and on, and check correctness under load, injected session stream faults, worker restarts, and backend restarts. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/durabletaskscheduler/dts_stress_test.go | 1228 +++++++++++++++++ 1 file changed, 1228 insertions(+) create mode 100644 tests/durabletaskscheduler/dts_stress_test.go diff --git a/tests/durabletaskscheduler/dts_stress_test.go b/tests/durabletaskscheduler/dts_stress_test.go new file mode 100644 index 00000000..8422f211 --- /dev/null +++ b/tests/durabletaskscheduler/dts_stress_test.go @@ -0,0 +1,1228 @@ +//go:build stress && unix + +// Stress and performance tests for extended sessions. They are not part of +// normal CI. Run them by hand against a session-enabled DTS endpoint: +// +// DTS_EMULATOR_ENDPOINT=http://127.0.0.1:8080 \ +// DTS_STRESS_RESTART=/path/to/restart-backend.sh \ +// go test -tags stress -run 'TestDTSPerf|TestDTSStress' -timeout 6h -v ./tests/durabletaskscheduler +// +// Settings: +// - DTS_STRESS_RESTART: a shell command that restarts the backend. The +// harness then waits until the endpoint answers. The perf test restarts +// the backend before every run. The backend-restart stress test needs it. +// - DTS_STRESS_PERSISTENT=1: the backend keeps its state across restarts, +// so the backend-restart test checks every instance and not only new ones. +// - DTS_STRESS_SCALE: multiplies instance, step, and signal counts (default 1). +// - DTS_STRESS_REPEAT: perf repetitions. The table shows the median (default 3). +// - DTS_STRESS_SESSIONS=off: run the stress tests without extended sessions. +// - DTS_STRESS_FAULT_RATE: the chance of a fault on each session stream in +// the chaos test (default 0.1). +// - DTS_STRESS_VERBOSE=1: print worker warnings, not only errors. +// +// Use 127.0.0.1 and not localhost. On some machines every localhost +// connection waits for an IPv6 attempt first. +// +// The CPU and memory numbers are for the whole test process, so they also +// include the client. + +package durabletaskscheduler_test + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "math" + "math/rand/v2" + "os" + "os/exec" + "runtime" + "runtime/pprof" + "slices" + "strconv" + "strings" + "sync" + "sync/atomic" + "syscall" + "testing" + "time" + + "github.com/google/uuid" + "github.com/microsoft/durabletask-go/api" + durabletaskclient "github.com/microsoft/durabletask-go/client" + "github.com/microsoft/durabletask-go/durabletaskscheduler" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/microsoft/durabletask-go/task" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" +) + +type stressFault int + +const ( + noFault stressFault = iota + faultCutBeforeAck + faultDropAck + faultCutMidBatch + faultCutIdle + faultDelayIdle + stressFaultKinds +) + +var stressFaultNames = [stressFaultKinds]string{ + "none", "cut before ack", "drop ack", "cut mid-batch", "cut while idle", "delay past idle timeout", +} + +// stressProbe counts what the worker sends and receives, and can inject +// faults on session streams. +type stressProbe struct { + faultRate float64 + idleDelay time.Duration + + historyBytes atomic.Int64 + workItems atomic.Int64 + replays atomic.Int64 + resumes atomic.Int64 + acks atomic.Int64 + unary atomic.Int64 + abandons atomic.Int64 + releases atomic.Int64 + notFound atomic.Int64 + open atomic.Int64 + maxOpen atomic.Int64 + faults [stressFaultKinds]atomic.Int64 + + mu sync.Mutex + rng *rand.Rand + turnStart map[string]time.Time + latencies []time.Duration +} + +func newStressProbe(faultRate float64, idleDelay time.Duration) *stressProbe { + return &stressProbe{ + faultRate: faultRate, + idleDelay: idleDelay, + rng: rand.New(rand.NewPCG(uint64(time.Now().UnixNano()), 1)), + turnStart: make(map[string]time.Time), + } +} + +func (p *stressProbe) pickFault() stressFault { + if p.faultRate <= 0 { + return noFault + } + p.mu.Lock() + defer p.mu.Unlock() + if p.rng.Float64() >= p.faultRate { + return noFault + } + return stressFault(1 + p.rng.IntN(int(stressFaultKinds)-1)) +} + +func (p *stressProbe) recordErr(err error) { + if status.Code(err) == codes.NotFound { + p.notFound.Add(1) + } +} + +// startTurn marks the time when the input of a turn is ready. +func (p *stressProbe) startTurn(token string) { + p.mu.Lock() + p.turnStart[token] = time.Now() + p.mu.Unlock() +} + +// finishTurn records the worker time of a turn when its completion is sent. +func (p *stressProbe) finishTurn(token string) { + p.mu.Lock() + defer p.mu.Unlock() + if start, ok := p.turnStart[token]; ok { + p.latencies = append(p.latencies, time.Since(start)) + delete(p.turnStart, token) + } +} + +func (p *stressProbe) faultCount() int64 { + var total int64 + for i := range p.faults { + total += p.faults[i].Load() + } + return total +} + +func (p *stressProbe) streamInterceptor() grpc.StreamClientInterceptor { + return func( + ctx context.Context, + desc *grpc.StreamDesc, + cc *grpc.ClientConn, + method string, + streamer grpc.Streamer, + opts ...grpc.CallOption, + ) (grpc.ClientStream, error) { + switch method { + case protos.TaskHubSidecarService_GetWorkItems_FullMethodName, + protos.TaskHubSidecarService_StreamInstanceHistory_FullMethodName: + stream, err := streamer(ctx, desc, cc, method, opts...) + if err != nil { + p.recordErr(err) + return nil, err + } + return &probedReadStream{ClientStream: stream, probe: p}, nil + case protos.TaskHubSidecarService_CompleteOrchestratorTaskWithExtendedSession_FullMethodName, + protos.TaskHubSidecarService_CompleteEntityTaskWithExtendedSession_FullMethodName: + streamCtx, cancel := context.WithCancel(ctx) + stream, err := streamer(streamCtx, desc, cc, method, opts...) + if err != nil { + cancel() + p.recordErr(err) + return nil, err + } + open := p.open.Add(1) + for current := p.maxOpen.Load(); open > current && !p.maxOpen.CompareAndSwap(current, open); current = p.maxOpen.Load() { + } + s := &probedSessionStream{ClientStream: stream, probe: p, ctx: streamCtx, cancel: cancel, fault: p.pickFault()} + context.AfterFunc(streamCtx, s.finish) + return s, nil + default: + return streamer(ctx, desc, cc, method, opts...) + } + } +} + +func (p *stressProbe) unaryInterceptor() grpc.UnaryClientInterceptor { + return func( + ctx context.Context, + method string, + req, reply any, + cc *grpc.ClientConn, + invoker grpc.UnaryInvoker, + opts ...grpc.CallOption, + ) error { + switch method { + case protos.TaskHubSidecarService_CompleteOrchestratorTask_FullMethodName: + p.unary.Add(1) + p.finishTurn(req.(*protos.OrchestratorResponse).GetCompletionToken()) + case protos.TaskHubSidecarService_CompleteEntityTask_FullMethodName: + p.unary.Add(1) + p.finishTurn(req.(*protos.EntityBatchResult).GetCompletionToken()) + case protos.TaskHubSidecarService_AbandonTaskOrchestratorWorkItem_FullMethodName, + protos.TaskHubSidecarService_AbandonTaskEntityWorkItem_FullMethodName, + protos.TaskHubSidecarService_AbandonTaskActivityWorkItem_FullMethodName: + p.abandons.Add(1) + case protos.TaskHubSidecarService_ReleaseTaskOrchestratorWorkItem_FullMethodName, + protos.TaskHubSidecarService_ReleaseTaskEntityWorkItem_FullMethodName: + p.releases.Add(1) + } + err := invoker(ctx, method, req, reply, cc, opts...) + p.recordErr(err) + return err + } +} + +type probedReadStream struct { + grpc.ClientStream + probe *stressProbe +} + +func (s *probedReadStream) RecvMsg(m any) error { + err := s.ClientStream.RecvMsg(m) + if err != nil { + s.probe.recordErr(err) + return err + } + switch message := m.(type) { + case *protos.HistoryChunk: + s.probe.historyBytes.Add(int64(proto.Size(message))) + case *protos.WorkItem: + switch { + case message.GetOrchestratorRequest() != nil: + request := message.GetOrchestratorRequest() + if len(request.GetPastEvents()) > 0 || request.GetRequiresHistoryStreaming() { + s.probe.replays.Add(1) + } + case message.GetEntityRequestV2() != nil, message.GetEntityRequest() != nil: + default: + return nil + } + s.probe.workItems.Add(1) + s.probe.historyBytes.Add(int64(proto.Size(message))) + s.probe.startTurn(message.GetCompletionToken()) + } + return nil +} + +type probedSessionStream struct { + grpc.ClientStream + probe *stressProbe + ctx context.Context + cancel context.CancelFunc + fault stressFault + once sync.Once + token string + batch bool + broken bool + delay bool +} + +func (s *probedSessionStream) finish() { + s.once.Do(func() { s.probe.open.Add(-1) }) +} + +// inject breaks the stream on both sides, so the service also sees it end. +func (s *probedSessionStream) inject() error { + s.probe.faults[s.fault].Add(1) + s.cancel() + s.finish() + return status.Error(codes.Unavailable, "injected: "+stressFaultNames[s.fault]) +} + +func (s *probedSessionStream) SendMsg(m any) error { + if s.fault == faultCutBeforeAck { + return s.inject() + } + switch request := m.(type) { + case *protos.OrchestratorResponse: + s.token = request.GetCompletionToken() + case *protos.EntityBatchResult: + s.token = request.GetCompletionToken() + } + s.probe.finishTurn(s.token) + return s.ClientStream.SendMsg(m) +} + +func (s *probedSessionStream) RecvMsg(m any) error { + if s.broken { + return s.inject() + } + if s.delay { + s.delay = false + s.probe.faults[faultDelayIdle].Add(1) + select { + case <-time.After(s.probe.idleDelay): + case <-s.ctx.Done(): + s.finish() + return status.FromContextError(s.ctx.Err()).Err() + } + } + err := s.ClientStream.RecvMsg(m) + if err != nil { + if errors.Is(err, io.EOF) && s.batch { + s.probe.resumes.Add(1) + s.probe.startTurn(s.token) + } + s.probe.recordErr(err) + s.cancel() + s.finish() + return err + } + response := m.(*protos.CompleteTaskWithExtendedSessionResponse) + switch { + case response.GetAck() != nil: + if s.fault == faultDropAck { + return s.inject() + } + s.probe.acks.Add(1) + s.broken = s.fault == faultCutIdle + s.delay = s.fault == faultDelayIdle + case response.GetNewMessages() != nil: + s.probe.historyBytes.Add(int64(proto.Size(response.GetNewMessages()))) + s.batch = true + s.broken = s.fault == faultCutMidBatch + } + return nil +} + +func envFloat(name string, fallback float64) float64 { + if value, err := strconv.ParseFloat(os.Getenv(name), 64); err == nil { + return value + } + return fallback +} + +func scaled(n int) int { + return max(1, int(math.Round(float64(n)*envFloat("DTS_STRESS_SCALE", 1)))) +} + +func stressLogger() api.Logger { + level := slog.LevelError + if os.Getenv("DTS_STRESS_VERBOSE") == "1" { + level = slog.LevelWarn + } + return api.NewSlogLogger(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: level}))) +} + +// restartBackend runs DTS_STRESS_RESTART and waits until the endpoint +// answers. It does nothing when no restart command is set. +func restartBackend(t *testing.T) error { + command := os.Getenv("DTS_STRESS_RESTART") + if command == "" { + return nil + } + if output, err := exec.Command("sh", "-c", command).CombinedOutput(); err != nil { + return fmt.Errorf("restart command failed: %w: %s", err, output) + } + client, err := durabletaskscheduler.NewClient(context.Background(), emulatorOptions(t), stressLogger()) + if err != nil { + return err + } + defer func() { _ = client.Close() }() + for deadline := time.Now().Add(3 * time.Minute); time.Now().Before(deadline); time.Sleep(500 * time.Millisecond) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + _, err = client.FetchOrchestrationMetadata(ctx, api.InstanceID("stress-ready-"+uuid.NewString())) + cancel() + if err == nil || errors.Is(err, api.ErrInstanceNotFound) { + return nil + } + } + return fmt.Errorf("the backend did not answer after the restart: %w", err) +} + +// stressHarness owns one management client and a worker that can be +// replaced while the test runs. +type stressHarness struct { + t *testing.T + registry *task.TaskRegistry + probe *stressProbe + workerOptions []durabletaskclient.TaskHubGrpcWorkerOption + client *durabletaskscheduler.Client + + mu sync.Mutex + worker *durabletaskclient.TaskHubGrpcWorker + restarts int +} + +func newStressHarness( + t *testing.T, + registry *task.TaskRegistry, + probe *stressProbe, + sessions *durabletaskclient.ExtendedSessionOptions, + workerOptions ...durabletaskclient.TaskHubGrpcWorkerOption, +) *stressHarness { + t.Helper() + client, err := durabletaskscheduler.NewClient(context.Background(), emulatorOptions(t), stressLogger()) + require.NoError(t, err) + options := append([]durabletaskclient.TaskHubGrpcWorkerOption{ + durabletaskclient.WithAutoWorkItemFilters(), + durabletaskclient.WithWorkerSilentDisconnectTimeout(15 * time.Second), + }, workerOptions...) + if sessions != nil { + options = append(options, durabletaskclient.WithExtendedSessions(*sessions)) + } + h := &stressHarness{t: t, registry: registry, probe: probe, workerOptions: options, client: client} + require.NoError(t, h.startWorker()) + return h +} + +func (h *stressHarness) startWorker() error { + options := emulatorOptions(h.t) + options.StreamInterceptors = append(slices.Clone(options.StreamInterceptors), h.probe.streamInterceptor()) + options.UnaryInterceptors = append(slices.Clone(options.UnaryInterceptors), h.probe.unaryInterceptor()) + worker, err := durabletaskscheduler.NewWorker(options, h.registry, stressLogger(), h.workerOptions...) + if err != nil { + return err + } + if err := worker.Start(context.Background()); err != nil { + return err + } + h.mu.Lock() + h.worker = worker + h.mu.Unlock() + return nil +} + +func (h *stressHarness) stopWorker() { + h.mu.Lock() + worker := h.worker + h.worker = nil + h.mu.Unlock() + if worker == nil { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if err := worker.Shutdown(ctx); err != nil { + h.t.Logf("worker shutdown: %v", err) + } +} + +// restartWorker replaces the worker. It is safe to call from any goroutine. +func (h *stressHarness) restartWorker() { + h.stopWorker() + assert.NoError(h.t, h.startWorker()) + h.mu.Lock() + h.restarts++ + h.mu.Unlock() +} + +func (h *stressHarness) close() { + h.stopWorker() + require.NoError(h.t, h.client.Close()) +} + +func stressRegistry(t *testing.T) *task.TaskRegistry { + t.Helper() + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddActivityN("PerfStep", func(ctx task.ActivityContext) (any, error) { + var input int + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + return input + 1, nil + })) + require.NoError(t, registry.AddOrchestratorN("PerfChain", func(ctx *task.OrchestrationContext) (any, error) { + var steps int + if err := ctx.GetInput(&steps); err != nil { + return nil, err + } + value := 0 + for range steps { + if err := ctx.CallActivity("PerfStep", task.WithActivityInput(value)).Await(&value); err != nil { + return nil, err + } + } + return value, nil + })) + require.NoError(t, registry.AddOrchestratorN("PerfEvents", func(ctx *task.OrchestrationContext) (any, error) { + var events int + if err := ctx.GetInput(&events); err != nil { + return nil, err + } + sum := 0 + for i := range events { + var value int + if err := ctx.WaitForSingleEvent("e", -1).Await(&value); err != nil { + return nil, err + } + sum += value + if err := ctx.SetCustomStatusValue(i + 1); err != nil { + return nil, err + } + } + return sum, nil + })) + require.NoError(t, registry.AddOrchestratorN("PerfFanOut", func(ctx *task.OrchestrationContext) (any, error) { + var width int + if err := ctx.GetInput(&width); err != nil { + return nil, err + } + return fanOut(ctx, 0, width) + })) + require.NoError(t, registry.AddEntityN("StressCounter", func(ctx *task.EntityContext) (any, error) { + var value int + if ctx.HasState() { + if err := ctx.GetState(&value); err != nil { + return nil, err + } + } + if ctx.Operation != "add" { + return nil, fmt.Errorf("unknown operation %q", ctx.Operation) + } + var amount int + if err := ctx.GetInput(&amount); err != nil { + return nil, err + } + value += amount + return value, ctx.SetState(value) + })) + require.NoError(t, registry.AddOrchestratorN("StressMixed", stressMixed)) + return registry +} + +func fanOut(ctx *task.OrchestrationContext, base, width int) (int, error) { + tasks := make([]task.Task, width) + for i := range tasks { + tasks[i] = ctx.CallActivity("PerfStep", task.WithActivityInput(base+i)) + } + if err := ctx.WhenAll(tasks...); err != nil { + return 0, err + } + sum := 0 + for _, pending := range tasks { + var value int + if err := pending.Await(&value); err != nil { + return 0, err + } + sum += value + } + return sum, nil +} + +type stressInput struct { + Index int + Entities int + Prefix string +} + +const stressEventsPerInstance = 3 + +func stressEntityID(prefix string, index int) api.EntityID { + return api.NewEntityID("StressCounter", fmt.Sprintf("%s-%d", prefix, index)) +} + +// stressMixed runs one of six shapes, chosen by the instance index. +// stressExpected returns its result. +func stressMixed(ctx *task.OrchestrationContext) (any, error) { + var input stressInput + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + i := input.Index + switch i % 6 { + case 0: + value := i + for range 3 + i%5 { + if err := ctx.CallActivity("PerfStep", task.WithActivityInput(value)).Await(&value); err != nil { + return nil, err + } + } + return value, nil + case 1: + return fanOut(ctx, i, 10) + case 2: + sum := 0 + for range stressEventsPerInstance { + var value int + if err := ctx.WaitForSingleEvent("e", 30*time.Minute).Await(&value); err != nil { + return nil, err + } + sum += value + } + return sum, nil + case 3: + sum := 0 + for range 2 { + var value int + if err := ctx.CallSubOrchestrator("PerfChain", task.WithSubOrchestratorInput(3)).Await(&value); err != nil { + return nil, err + } + sum += value + } + return sum, nil + case 4: + if err := ctx.CreateTimer(50 * time.Millisecond).Await(nil); err != nil { + return nil, err + } + var value int + err := ctx.CallActivity("PerfStep", task.WithActivityInput(i)).Await(&value) + return value, err + default: + entityID := stressEntityID(input.Prefix, i%input.Entities) + if i%2 == 1 { + unlock, err := ctx.LockEntities(entityID) + if err != nil { + return nil, err + } + defer unlock() + } + return 1, ctx.CallEntity(entityID, "add", task.WithEntityInput(1)).Await(nil) + } +} + +func stressExpected(i int) int { + switch i % 6 { + case 0: + return i + 3 + i%5 + case 1: + return 10*i + 45 + 10 + case 2: + return stressEventsPerInstance * i + case 3: + return 6 + case 4: + return i + 1 + default: + return 1 + } +} + +// ---------------------------------------------------------------------------- +// Performance + +type perfResult struct { + wall time.Duration + units int + cpu time.Duration + totalAlloc uint64 + peakHeap uint64 + historyBytes int64 + workItems int64 + replays int64 + resumes int64 + acks int64 + unary int64 + p50 time.Duration + p95 time.Duration + p99 time.Duration +} + +type perfWorkload struct { + name string + units string + run func(t *testing.T, ctx context.Context, client *durabletaskscheduler.Client) int +} + +func processCPU() time.Duration { + var usage syscall.Rusage + if err := syscall.Getrusage(syscall.RUSAGE_SELF, &usage); err != nil { + return 0 + } + return time.Duration(usage.Utime.Nano() + usage.Stime.Nano()) +} + +func percentile(sorted []time.Duration, q float64) time.Duration { + if len(sorted) == 0 { + return 0 + } + return sorted[min(len(sorted)-1, int(math.Ceil(q*float64(len(sorted))))-1)] +} + +// waitCompleted is safe to call from any goroutine. +func waitCompleted(t *testing.T, ctx context.Context, client *durabletaskscheduler.Client, id api.InstanceID, want int) { + metadata, err := client.WaitForOrchestrationCompletion(ctx, id, api.WithFetchPayloads(true)) + if !assert.NoError(t, err) { + return + } + var got int + if assert.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, metadata.RuntimeStatus, "failure: %v", metadata.FailureDetails) && + assert.NoError(t, metadata.ReadOutput(&got)) { + assert.Equal(t, want, got, "%s output", id) + } +} + +func perfWorkloads() []perfWorkload { + return []perfWorkload{ + {name: "chain", units: "activities", run: func(t *testing.T, ctx context.Context, client *durabletaskscheduler.Client) int { + steps := scaled(500) + id, err := client.ScheduleNewOrchestration(ctx, "PerfChain", api.WithInput(steps)) + require.NoError(t, err) + waitCompleted(t, ctx, client, id, steps) + return steps + }}, + {name: "events", units: "events", run: func(t *testing.T, ctx context.Context, client *durabletaskscheduler.Client) int { + events := scaled(200) + id, err := client.ScheduleNewOrchestration(ctx, "PerfEvents", api.WithInput(events)) + require.NoError(t, err) + for i := range events { + require.NoError(t, client.RaiseEvent(ctx, id, "e", api.WithEventPayload(i))) + // Raise the next event only after this one was processed, so + // every event is its own turn. + for { + metadata, err := client.FetchOrchestrationMetadata(ctx, id, api.WithFetchPayloads(true)) + require.NoError(t, err) + var processed int + if metadata.SerializedCustomStatus != "" { + require.NoError(t, metadata.ReadCustomStatus(&processed)) + } + if processed > i { + break + } + time.Sleep(time.Millisecond) + } + } + waitCompleted(t, ctx, client, id, events*(events-1)/2) + return events + }}, + {name: "fan-out", units: "activities", run: func(t *testing.T, ctx context.Context, client *durabletaskscheduler.Client) int { + width := scaled(200) + id, err := client.ScheduleNewOrchestration(ctx, "PerfFanOut", api.WithInput(width)) + require.NoError(t, err) + waitCompleted(t, ctx, client, id, width*(width+1)/2) + return width + }}, + {name: "entity", units: "signals", run: func(t *testing.T, ctx context.Context, client *durabletaskscheduler.Client) int { + signals := scaled(1000) + entityID := stressEntityID("perf-"+uuid.NewString(), 0) + for range signals { + require.NoError(t, client.SignalEntity(ctx, entityID, "add", api.WithSignalInput(1))) + } + want := strconv.Itoa(signals) + require.Eventually(t, func() bool { + metadata, err := client.GetEntity(ctx, entityID) + return err == nil && metadata != nil && metadata.SerializedState == want + }, 10*time.Minute, 5*time.Millisecond) + return signals + }}, + {name: "short", units: "activities", run: func(t *testing.T, ctx context.Context, client *durabletaskscheduler.Client) int { + instances := scaled(1000) + ids := make([]api.InstanceID, instances) + runParallel(instances, 32, func(i int) { + id, err := client.ScheduleNewOrchestration(ctx, "PerfChain", api.WithInput(5)) + assert.NoError(t, err) + ids[i] = id + }) + require.False(t, t.Failed()) + runParallel(instances, 32, func(i int) { waitCompleted(t, ctx, client, ids[i], 5) }) + return 5 * instances + }}, + } +} + +func runParallel(n, workers int, fn func(int)) { + var next atomic.Int64 + var wg sync.WaitGroup + for range min(n, workers) { + wg.Go(func() { + for i := int(next.Add(1)) - 1; i < n; i = int(next.Add(1)) - 1 { + fn(i) + } + }) + } + wg.Wait() +} + +func runPerf(t *testing.T, workload perfWorkload, sessions bool) perfResult { + t.Helper() + var sessionOptions *durabletaskclient.ExtendedSessionOptions + if sessions { + sessionOptions = &durabletaskclient.ExtendedSessionOptions{} + } + probe := newStressProbe(0, 0) + h := newStressHarness(t, stressRegistry(t), probe, sessionOptions, + durabletaskclient.WithMaxConcurrentOrchestrationWorkItems(16), + durabletaskclient.WithMaxConcurrentActivityWorkItems(32), + durabletaskclient.WithMaxConcurrentEntityWorkItems(16), + ) + defer h.close() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) + defer cancel() + + runtime.GC() + var before runtime.MemStats + runtime.ReadMemStats(&before) + var peakHeap atomic.Uint64 + stopSampler := make(chan struct{}) + sampled := make(chan struct{}) + go func() { + defer close(sampled) + ticker := time.NewTicker(20 * time.Millisecond) + defer ticker.Stop() + for { + var stats runtime.MemStats + runtime.ReadMemStats(&stats) + if stats.HeapInuse > peakHeap.Load() { + peakHeap.Store(stats.HeapInuse) + } + select { + case <-stopSampler: + return + case <-ticker.C: + } + } + }() + cpuStart := processCPU() + start := time.Now() + units := workload.run(t, ctx, h.client) + wall := time.Since(start) + cpu := processCPU() - cpuStart + close(stopSampler) + <-sampled + var after runtime.MemStats + runtime.ReadMemStats(&after) + + probe.mu.Lock() + latencies := slices.Clone(probe.latencies) + probe.mu.Unlock() + slices.Sort(latencies) + if sessions { + require.Positive(t, probe.acks.Load(), "the sessions-on run did not use sessions") + } else { + require.Zero(t, probe.acks.Load(), "the sessions-off run used sessions") + } + return perfResult{ + wall: wall, + units: units, + cpu: cpu, + totalAlloc: after.TotalAlloc - before.TotalAlloc, + peakHeap: peakHeap.Load(), + historyBytes: probe.historyBytes.Load(), + workItems: probe.workItems.Load(), + replays: probe.replays.Load(), + resumes: probe.resumes.Load(), + acks: probe.acks.Load(), + unary: probe.unary.Load(), + p50: percentile(latencies, 0.50), + p95: percentile(latencies, 0.95), + p99: percentile(latencies, 0.99), + } +} + +func medianOf[T int64 | uint64 | time.Duration](results []perfResult, field func(perfResult) T) T { + values := make([]T, len(results)) + for i, result := range results { + values[i] = field(result) + } + slices.Sort(values) + return values[len(values)/2] +} + +func formatPerfRows(workload perfWorkload, off, on []perfResult) string { + var b strings.Builder + row := func(metric string, offValue, onValue float64, format func(float64) string) { + change := "n/a" + if offValue != 0 { + change = fmt.Sprintf("%+.1f%%", 100*(onValue-offValue)/offValue) + } + fmt.Fprintf(&b, "| %s | %s | %s | %s | %s |\n", workload.name, metric, format(offValue), format(onValue), change) + } + ms := func(v float64) string { return fmt.Sprintf("%.2f ms", v/float64(time.Millisecond)) } + mib := func(v float64) string { + if v < 1<<20 { + return fmt.Sprintf("%.1f KiB", v/(1<<10)) + } + return fmt.Sprintf("%.1f MiB", v/(1<<20)) + } + count := func(v float64) string { return strconv.FormatFloat(v, 'f', 0, 64) } + perSecond := func(v float64) string { return fmt.Sprintf("%.0f %s/s", v, workload.units) } + durationRow := func(metric string, field func(perfResult) time.Duration) { + row(metric, float64(medianOf(off, field)), float64(medianOf(on, field)), ms) + } + intRow := func(metric string, field func(perfResult) int64, format func(float64) string) { + row(metric, float64(medianOf(off, field)), float64(medianOf(on, field)), format) + } + throughput := func(results []perfResult) float64 { + wall := medianOf(results, func(r perfResult) time.Duration { return r.wall }) + return float64(results[0].units) / wall.Seconds() + } + durationRow("wall time", func(r perfResult) time.Duration { return r.wall }) + row("throughput", throughput(off), throughput(on), perSecond) + durationRow("turn p50", func(r perfResult) time.Duration { return r.p50 }) + durationRow("turn p95", func(r perfResult) time.Duration { return r.p95 }) + durationRow("turn p99", func(r perfResult) time.Duration { return r.p99 }) + durationRow("process CPU", func(r perfResult) time.Duration { return r.cpu }) + row("allocated", float64(medianOf(off, func(r perfResult) uint64 { return r.totalAlloc })), + float64(medianOf(on, func(r perfResult) uint64 { return r.totalAlloc })), mib) + row("peak heap in use", float64(medianOf(off, func(r perfResult) uint64 { return r.peakHeap })), + float64(medianOf(on, func(r perfResult) uint64 { return r.peakHeap })), mib) + intRow("history received", func(r perfResult) int64 { return r.historyBytes }, mib) + intRow("work items (loads)", func(r perfResult) int64 { return r.workItems }, count) + intRow("replays", func(r perfResult) int64 { return r.replays }, count) + intRow("session resumes", func(r perfResult) int64 { return r.resumes }, count) + intRow("session turns (acks)", func(r perfResult) int64 { return r.acks }, count) + intRow("unary completions", func(r perfResult) int64 { return r.unary }, count) + return b.String() +} + +// TestDTSPerf compares the same workloads with sessions off and on. With +// DTS_STRESS_RESTART set, every run starts on a fresh backend. +func TestDTSPerf(t *testing.T) { + emulatorOptions(t) + repeat := max(1, int(envFloat("DTS_STRESS_REPEAT", 3))) + const header = "| workload | metric | sessions off | sessions on | change |\n|---|---|---|---|---|\n" + var table strings.Builder + for _, workload := range perfWorkloads() { + t.Run(workload.name, func(t *testing.T) { + results := map[bool][]perfResult{} + for range repeat { + for _, sessions := range []bool{false, true} { + require.NoError(t, restartBackend(t)) + result := runPerf(t, workload, sessions) + t.Logf("sessions=%v wall=%v acks=%d unary=%d replays=%d resumes=%d", + sessions, result.wall, result.acks, result.unary, result.replays, result.resumes) + results[sessions] = append(results[sessions], result) + } + } + rows := formatPerfRows(workload, results[false], results[true]) + t.Logf("median of %d runs:\n%s%s", repeat, header, rows) + table.WriteString(rows) + }) + } + t.Logf("median of %d runs:\n%s%s", repeat, header, table.String()) +} + +// ---------------------------------------------------------------------------- +// Stress + +type stressConfig struct { + instances int + entities int + signalsPerEntity int + orchestrations int + activities int + entityConcurrency int + completionConnections int + faultRate float64 + workerRestarts int + backendRestart bool +} + +type stressBatch struct { + prefix string + ids []api.InstanceID + entities int + expected map[int]int +} + +func stressSessionsOn() bool { + return os.Getenv("DTS_STRESS_SESSIONS") != "off" +} + +// scheduleStressBatch starts the instances, raises their events, and sends +// the entity signals. +func scheduleStressBatch(t *testing.T, ctx context.Context, client *durabletaskscheduler.Client, config stressConfig) *stressBatch { + t.Helper() + batch := &stressBatch{ + prefix: "stress-" + uuid.NewString()[:8], + ids: make([]api.InstanceID, config.instances), + entities: config.entities, + expected: make(map[int]int), + } + for i := range config.instances { + if i%6 == 5 { + batch.expected[i%config.entities]++ + } + } + for e := range config.entities { + batch.expected[e] += config.signalsPerEntity + } + runParallel(config.instances, 32, func(i int) { + id := api.InstanceID(fmt.Sprintf("%s-%d", batch.prefix, i)) + _, err := client.ScheduleNewOrchestration(ctx, "StressMixed", api.WithInstanceID(id), + api.WithInput(stressInput{Index: i, Entities: config.entities, Prefix: batch.prefix})) + assert.NoError(t, err) + batch.ids[i] = id + if i%6 == 2 { + for range stressEventsPerInstance { + assert.NoError(t, client.RaiseEvent(ctx, id, "e", api.WithEventPayload(i))) + } + } + }) + runParallel(config.entities*config.signalsPerEntity, 32, func(n int) { + assert.NoError(t, client.SignalEntity(ctx, stressEntityID(batch.prefix, n%config.entities), "add", api.WithSignalInput(1))) + }) + require.False(t, t.Failed(), "scheduling failed") + return batch +} + +// verifyStressBatch waits for every instance and entity and returns the +// problems it found. completed counts the finished instances. +func verifyStressBatch(ctx context.Context, client *durabletaskscheduler.Client, batch *stressBatch, completed *atomic.Int64) []string { + var mu sync.Mutex + var problems []string + report := func(format string, args ...any) { + mu.Lock() + problems = append(problems, fmt.Sprintf(format, args...)) + mu.Unlock() + } + runParallel(len(batch.ids), 32, func(i int) { + for { + metadata, err := client.WaitForOrchestrationCompletion(ctx, batch.ids[i], api.WithFetchPayloads(true)) + if err != nil { + if ctx.Err() != nil { + status := "unknown" + if latest, fetchErr := client.FetchOrchestrationMetadata(context.Background(), batch.ids[i]); fetchErr == nil && latest != nil { + status = latest.RuntimeStatus.String() + } + report("%s: not finished (%s): %v", batch.ids[i], status, err) + return + } + // The backend may be restarting. + time.Sleep(time.Second) + continue + } + completed.Add(1) + var got int + if metadata.RuntimeStatus != protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED { + report("%s: status %s: %v", batch.ids[i], metadata.RuntimeStatus, metadata.FailureDetails) + } else if err := metadata.ReadOutput(&got); err != nil || got != stressExpected(i) { + report("%s: output %d, want %d (%v)", batch.ids[i], got, stressExpected(i), err) + } + return + } + }) + // Signals can still be running after the last instance completes. + entityDeadline := time.Now().Add(3 * time.Minute) + for e := range batch.entities { + entityID := stressEntityID(batch.prefix, e) + want := strconv.Itoa(batch.expected[e]) + got := "" + for ctx.Err() == nil && time.Now().Before(entityDeadline) { + metadata, err := client.GetEntity(ctx, entityID) + if err == nil && metadata != nil { + got = metadata.SerializedState + } + if got == want { + break + } + time.Sleep(50 * time.Millisecond) + } + if got != want { + report("%s: state %q, want %s", entityID, got, want) + } + } + return problems +} + +func runStress(t *testing.T, config stressConfig) { + sessionsOn := stressSessionsOn() + if config.backendRestart && os.Getenv("DTS_STRESS_RESTART") == "" { + t.Skip("set DTS_STRESS_RESTART to run the backend-restart test") + } + baseline := runtime.NumGoroutine() + idleTimeout := 3 * time.Second + var sessions *durabletaskclient.ExtendedSessionOptions + if sessionsOn { + sessions = &durabletaskclient.ExtendedSessionOptions{IdleTimeout: idleTimeout} + } + probe := newStressProbe(config.faultRate, idleTimeout+time.Second) + workerOptions := []durabletaskclient.TaskHubGrpcWorkerOption{ + durabletaskclient.WithMaxConcurrentOrchestrationWorkItems(config.orchestrations), + durabletaskclient.WithMaxConcurrentActivityWorkItems(config.activities), + durabletaskclient.WithMaxConcurrentEntityWorkItems(config.entityConcurrency), + } + if config.completionConnections > 0 { + workerOptions = append(workerOptions, durabletaskclient.WithWorkerCompletionConnections(config.completionConnections)) + } + h := newStressHarness(t, stressRegistry(t), probe, sessions, workerOptions...) + closed := false + defer func() { + if !closed { + h.close() + } + }() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) + defer cancel() + + start := time.Now() + batch := scheduleStressBatch(t, ctx, h.client, config) + t.Logf("scheduled %d instances and %d signals in %v", config.instances, config.entities*config.signalsPerEntity, time.Since(start)) + + var completed atomic.Int64 + done := make(chan struct{}) + var background sync.WaitGroup + // waitProgress waits until a share of the instances completed. It + // returns false when the run ends first. + waitProgress := func(share float64) bool { + for float64(completed.Load()) < share*float64(config.instances) { + select { + case <-done: + return false + case <-time.After(10 * time.Millisecond): + } + } + return true + } + if config.workerRestarts > 0 { + background.Go(func() { + for n := 1; n <= config.workerRestarts; n++ { + if !waitProgress(float64(n) / float64(config.workerRestarts+1)) { + return + } + h.restartWorker() + } + }) + } + persistent := os.Getenv("DTS_STRESS_PERSISTENT") == "1" + if config.backendRestart { + background.Go(func() { + if !waitProgress(0.3) { + return + } + t.Logf("restarting the backend after %d of %d instances", completed.Load(), config.instances) + assert.NoError(t, restartBackend(t)) + t.Logf("the backend is back after %v", time.Since(start)) + }) + } + var problems []string + if config.backendRestart && !persistent { + // All state is lost on restart. Check only that new work completes. + for completed.Load() < int64(config.instances)*3/10 { + _, err := h.client.WaitForOrchestrationCompletion(ctx, batch.ids[completed.Load()]) + if err == nil { + completed.Add(1) + } + require.NoError(t, ctx.Err()) + } + background.Wait() + fresh := config + fresh.instances, fresh.entities, fresh.signalsPerEntity = scaled(120), scaled(12), 5 + freshBatch := scheduleStressBatch(t, ctx, h.client, fresh) + problems = verifyStressBatch(ctx, h.client, freshBatch, new(atomic.Int64)) + } else { + problems = verifyStressBatch(ctx, h.client, batch, &completed) + } + close(done) + background.Wait() + elapsed := time.Since(start) + + t.Logf("finished in %v: worker restarts=%d acks=%d unary=%d replays=%d resumes=%d releases=%d abandons=%d notFound=%d maxOpenStreams=%d", + elapsed, h.restarts, probe.acks.Load(), probe.unary.Load(), probe.replays.Load(), probe.resumes.Load(), + probe.releases.Load(), probe.abandons.Load(), probe.notFound.Load(), probe.maxOpen.Load()) + for kind := faultCutBeforeAck; kind < stressFaultKinds; kind++ { + if n := probe.faults[kind].Load(); n > 0 { + t.Logf("injected %q: %d", stressFaultNames[kind], n) + } + } + for _, problem := range problems[:min(len(problems), 20)] { + t.Error(problem) + } + require.Empty(t, problems, "%d problems", len(problems)) + + if sessionsOn { + require.Positive(t, probe.acks.Load(), "sessions were not used") + connections := config.completionConnections + if connections == 0 { + connections = 3 + } + require.LessOrEqual(t, probe.maxOpen.Load(), int64(50*connections), "a connection exceeded its session stream budget") + // Idle sessions end after the idle timeout and return their streams. + require.Eventually(t, func() bool { return probe.open.Load() == 0 }, idleTimeout+30*time.Second, 50*time.Millisecond, + "%d session streams are still open", probe.open.Load()) + } else { + require.Zero(t, probe.acks.Load(), "sessions were used while off") + } + h.close() + closed = true + require.Zero(t, probe.open.Load(), "session streams are open after shutdown") + // Poll here: assert.Eventually runs its condition in another goroutine. + for deadline := time.Now().Add(20 * time.Second); runtime.NumGoroutine() > baseline && time.Now().Before(deadline); { + time.Sleep(100 * time.Millisecond) + } + if runtime.NumGoroutine() > baseline { + var stacks strings.Builder + _ = pprof.Lookup("goroutine").WriteTo(&stacks, 1) + t.Fatalf("goroutines: %d after shutdown, %d before:\n%s", runtime.NumGoroutine(), baseline, stacks.String()) + } + if probe.faultCount() == 0 && h.restarts == 0 && !config.backendRestart { + assert.Zero(t, probe.abandons.Load(), "unexpected abandons") + assert.Zero(t, probe.notFound.Load(), "unexpected NotFound errors") + } +} + +// TestDTSStress checks correctness under load and faults. Every subtest +// checks results, entity values, open session streams, and goroutines. +func TestDTSStress(t *testing.T) { + emulatorOptions(t) + instances, entities := scaled(2000), scaled(200) + t.Run("load", func(t *testing.T) { + runStress(t, stressConfig{ + instances: instances, entities: entities, signalsPerEntity: 10, + orchestrations: 16, activities: 32, entityConcurrency: 16, + }) + }) + t.Run("one-completion-connection", func(t *testing.T) { + runStress(t, stressConfig{ + instances: instances, entities: entities, signalsPerEntity: 10, + orchestrations: 160, activities: 64, entityConcurrency: 120, completionConnections: 1, + }) + }) + t.Run("chaos", func(t *testing.T) { + runStress(t, stressConfig{ + instances: instances, entities: entities, signalsPerEntity: 10, + orchestrations: 32, activities: 32, entityConcurrency: 16, + faultRate: envFloat("DTS_STRESS_FAULT_RATE", 0.1), workerRestarts: 3, + }) + }) + t.Run("backend-restart", func(t *testing.T) { + runStress(t, stressConfig{ + instances: instances, entities: entities, signalsPerEntity: 10, + orchestrations: 32, activities: 32, entityConcurrency: 16, + faultRate: envFloat("DTS_STRESS_FAULT_RATE", 0.1) / 2, backendRestart: true, + }) + }) +} From 9e78e102d9b6ff832aa8a58b95057f67d210a51c Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Fri, 6 Nov 2026 18:57:03 -0800 Subject: [PATCH 11/21] Trace work items in the stress harness and use the default silent timeout DTS_STRESS_TRACE prints every work item, completion, abandon, release, and session stream event with its token. The 15 second silent timeout was shorter than the 30 second DTS health ping, so idle workers reconnected. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/durabletaskscheduler/dts_stress_test.go | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/tests/durabletaskscheduler/dts_stress_test.go b/tests/durabletaskscheduler/dts_stress_test.go index 8422f211..7be46187 100644 --- a/tests/durabletaskscheduler/dts_stress_test.go +++ b/tests/durabletaskscheduler/dts_stress_test.go @@ -19,6 +19,8 @@ // - DTS_STRESS_FAULT_RATE: the chance of a fault on each session stream in // the chaos test (default 0.1). // - DTS_STRESS_VERBOSE=1: print worker warnings, not only errors. +// - DTS_STRESS_TRACE=1: print every work item the worker receives and every +// completion, abandon, and release it sends, with the completion token. // // Use 127.0.0.1 and not localhost. On some machines every localhost // connection waits for an IPv6 attempt first. @@ -220,6 +222,9 @@ func (p *stressProbe) unaryInterceptor() grpc.UnaryClientInterceptor { p.releases.Add(1) } err := invoker(ctx, method, req, reply, cc, opts...) + if token, ok := req.(interface{ GetCompletionToken() string }); ok { + stressTracef("%s %s: %v", method[strings.LastIndex(method, "/")+1:], token.GetCompletionToken(), err) + } p.recordErr(err) return err } @@ -240,6 +245,9 @@ func (s *probedReadStream) RecvMsg(m any) error { case *protos.HistoryChunk: s.probe.historyBytes.Add(int64(proto.Size(message))) case *protos.WorkItem: + if request := message.GetActivityRequest(); request != nil { + stressTracef("received activity %s %s", message.GetCompletionToken(), request.GetOrchestrationInstance().GetInstanceId()) + } switch { case message.GetOrchestratorRequest() != nil: request := message.GetOrchestratorRequest() @@ -250,6 +258,8 @@ func (s *probedReadStream) RecvMsg(m any) error { default: return nil } + stressTracef("received %s %s%s", message.GetCompletionToken(), + message.GetOrchestratorRequest().GetInstanceId(), message.GetEntityRequestV2().GetInstanceId()) s.probe.workItems.Add(1) s.probe.historyBytes.Add(int64(proto.Size(message))) s.probe.startTurn(message.GetCompletionToken()) @@ -276,6 +286,7 @@ func (s *probedSessionStream) finish() { // inject breaks the stream on both sides, so the service also sees it end. func (s *probedSessionStream) inject() error { + stressTracef("session fault %s: %s", s.token, stressFaultNames[s.fault]) s.probe.faults[s.fault].Add(1) s.cancel() s.finish() @@ -293,6 +304,7 @@ func (s *probedSessionStream) SendMsg(m any) error { s.token = request.GetCompletionToken() } s.probe.finishTurn(s.token) + stressTracef("session send %s", s.token) return s.ClientStream.SendMsg(m) } @@ -317,6 +329,7 @@ func (s *probedSessionStream) RecvMsg(m any) error { s.probe.startTurn(s.token) } s.probe.recordErr(err) + stressTracef("session end %s batch=%v: %v", s.token, s.batch, err) s.cancel() s.finish() return err @@ -338,6 +351,14 @@ func (s *probedSessionStream) RecvMsg(m any) error { return nil } +var stressTrace = os.Getenv("DTS_STRESS_TRACE") == "1" + +func stressTracef(format string, args ...any) { + if stressTrace { + fmt.Fprintf(os.Stderr, "TRACE %s "+format+"\n", append([]any{time.Now().Format("15:04:05.000000")}, args...)...) + } +} + func envFloat(name string, fallback float64) float64 { if value, err := strconv.ParseFloat(os.Getenv(name), 64); err == nil { return value @@ -407,10 +428,7 @@ func newStressHarness( t.Helper() client, err := durabletaskscheduler.NewClient(context.Background(), emulatorOptions(t), stressLogger()) require.NoError(t, err) - options := append([]durabletaskclient.TaskHubGrpcWorkerOption{ - durabletaskclient.WithAutoWorkItemFilters(), - durabletaskclient.WithWorkerSilentDisconnectTimeout(15 * time.Second), - }, workerOptions...) + options := append([]durabletaskclient.TaskHubGrpcWorkerOption{durabletaskclient.WithAutoWorkItemFilters()}, workerOptions...) if sessions != nil { options = append(options, durabletaskclient.WithExtendedSessions(*sessions)) } From 14d00267bf4098c9c8f2c970b9ea5e4a16b1d019 Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Fri, 6 Nov 2026 19:41:24 -0800 Subject: [PATCH 12/21] Tighten the stress harness checks Use one integer threshold for the backend restart, check instances that finish before a non-persistent restart, run the unlocked entity shape, assert every chaos fault kind and bounded abandons and NotFound errors, require serial perf workloads to resume sessions, fail on worker shutdown errors, alternate the perf order, report heap growth, and use clearer fault and payload names. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/durabletaskscheduler/dts_stress_test.go | 139 ++++++++++++------ 1 file changed, 90 insertions(+), 49 deletions(-) diff --git a/tests/durabletaskscheduler/dts_stress_test.go b/tests/durabletaskscheduler/dts_stress_test.go index 7be46187..04c4cfb2 100644 --- a/tests/durabletaskscheduler/dts_stress_test.go +++ b/tests/durabletaskscheduler/dts_stress_test.go @@ -69,25 +69,25 @@ type stressFault int const ( noFault stressFault = iota - faultCutBeforeAck + faultCutBeforeSend faultDropAck faultCutMidBatch - faultCutIdle - faultDelayIdle + faultCutAfterAck + faultDelayAfterAck stressFaultKinds ) var stressFaultNames = [stressFaultKinds]string{ - "none", "cut before ack", "drop ack", "cut mid-batch", "cut while idle", "delay past idle timeout", + "none", "cut-before-send", "drop-ack", "cut-mid-batch", "cut-after-ack", "delay-after-ack", } // stressProbe counts what the worker sends and receives, and can inject // faults on session streams. type stressProbe struct { - faultRate float64 - idleDelay time.Duration + faultRate float64 + idleTimeout time.Duration - historyBytes atomic.Int64 + payloadBytes atomic.Int64 workItems atomic.Int64 replays atomic.Int64 resumes atomic.Int64 @@ -106,12 +106,12 @@ type stressProbe struct { latencies []time.Duration } -func newStressProbe(faultRate float64, idleDelay time.Duration) *stressProbe { +func newStressProbe(faultRate float64, idleTimeout time.Duration) *stressProbe { return &stressProbe{ - faultRate: faultRate, - idleDelay: idleDelay, - rng: rand.New(rand.NewPCG(uint64(time.Now().UnixNano()), 1)), - turnStart: make(map[string]time.Time), + faultRate: faultRate, + idleTimeout: idleTimeout, + rng: rand.New(rand.NewPCG(uint64(time.Now().UnixNano()), 1)), + turnStart: make(map[string]time.Time), } } @@ -243,7 +243,7 @@ func (s *probedReadStream) RecvMsg(m any) error { } switch message := m.(type) { case *protos.HistoryChunk: - s.probe.historyBytes.Add(int64(proto.Size(message))) + s.probe.payloadBytes.Add(int64(proto.Size(message))) case *protos.WorkItem: if request := message.GetActivityRequest(); request != nil { stressTracef("received activity %s %s", message.GetCompletionToken(), request.GetOrchestrationInstance().GetInstanceId()) @@ -261,7 +261,7 @@ func (s *probedReadStream) RecvMsg(m any) error { stressTracef("received %s %s%s", message.GetCompletionToken(), message.GetOrchestratorRequest().GetInstanceId(), message.GetEntityRequestV2().GetInstanceId()) s.probe.workItems.Add(1) - s.probe.historyBytes.Add(int64(proto.Size(message))) + s.probe.payloadBytes.Add(int64(proto.Size(message))) s.probe.startTurn(message.GetCompletionToken()) } return nil @@ -294,7 +294,7 @@ func (s *probedSessionStream) inject() error { } func (s *probedSessionStream) SendMsg(m any) error { - if s.fault == faultCutBeforeAck { + if s.fault == faultCutBeforeSend { return s.inject() } switch request := m.(type) { @@ -313,11 +313,21 @@ func (s *probedSessionStream) RecvMsg(m any) error { return s.inject() } if s.delay { + // Hold back the next message past the worker's idle timeout. The + // worker usually ends the stream first. The fault counts only when the + // hold lasted at least the idle timeout. s.delay = false - s.probe.faults[faultDelayIdle].Add(1) + start := time.Now() + var cancelled bool select { - case <-time.After(s.probe.idleDelay): + case <-time.After(s.probe.idleTimeout + time.Second): case <-s.ctx.Done(): + cancelled = true + } + if time.Since(start) >= s.probe.idleTimeout { + s.probe.faults[faultDelayAfterAck].Add(1) + } + if cancelled { s.finish() return status.FromContextError(s.ctx.Err()).Err() } @@ -341,10 +351,10 @@ func (s *probedSessionStream) RecvMsg(m any) error { return s.inject() } s.probe.acks.Add(1) - s.broken = s.fault == faultCutIdle - s.delay = s.fault == faultDelayIdle + s.broken = s.fault == faultCutAfterAck + s.delay = s.fault == faultDelayAfterAck case response.GetNewMessages() != nil: - s.probe.historyBytes.Add(int64(proto.Size(response.GetNewMessages()))) + s.probe.payloadBytes.Add(int64(proto.Size(response.GetNewMessages()))) s.batch = true s.broken = s.fault == faultCutMidBatch } @@ -464,9 +474,7 @@ func (h *stressHarness) stopWorker() { } ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - if err := worker.Shutdown(ctx); err != nil { - h.t.Logf("worker shutdown: %v", err) - } + assert.NoError(h.t, worker.Shutdown(ctx), "worker shutdown") } // restartWorker replaces the worker. It is safe to call from any goroutine. @@ -631,7 +639,8 @@ func stressMixed(ctx *task.OrchestrationContext) (any, error) { return value, err default: entityID := stressEntityID(input.Prefix, i%input.Entities) - if i%2 == 1 { + // i%6 == 5 here, so i is always odd. Use i/6 to alternate the shapes. + if (i/6)%2 == 1 { unlock, err := ctx.LockEntities(entityID) if err != nil { return nil, err @@ -668,7 +677,7 @@ type perfResult struct { cpu time.Duration totalAlloc uint64 peakHeap uint64 - historyBytes int64 + payloadBytes int64 workItems int64 replays int64 resumes int64 @@ -682,7 +691,10 @@ type perfResult struct { type perfWorkload struct { name string units string - run func(t *testing.T, ctx context.Context, client *durabletaskscheduler.Client) int + // serial workloads drive one instance through many turns, so with + // sessions on it must load once and resume for almost every turn. + serial bool + run func(t *testing.T, ctx context.Context, client *durabletaskscheduler.Client) int } func processCPU() time.Duration { @@ -715,14 +727,14 @@ func waitCompleted(t *testing.T, ctx context.Context, client *durabletaskschedul func perfWorkloads() []perfWorkload { return []perfWorkload{ - {name: "chain", units: "activities", run: func(t *testing.T, ctx context.Context, client *durabletaskscheduler.Client) int { + {name: "chain", units: "activities", serial: true, run: func(t *testing.T, ctx context.Context, client *durabletaskscheduler.Client) int { steps := scaled(500) id, err := client.ScheduleNewOrchestration(ctx, "PerfChain", api.WithInput(steps)) require.NoError(t, err) waitCompleted(t, ctx, client, id, steps) return steps }}, - {name: "events", units: "events", run: func(t *testing.T, ctx context.Context, client *durabletaskscheduler.Client) int { + {name: "events", units: "events", serial: true, run: func(t *testing.T, ctx context.Context, client *durabletaskscheduler.Client) int { events := scaled(200) id, err := client.ScheduleNewOrchestration(ctx, "PerfEvents", api.WithInput(events)) require.NoError(t, err) @@ -753,7 +765,7 @@ func perfWorkloads() []perfWorkload { waitCompleted(t, ctx, client, id, width*(width+1)/2) return width }}, - {name: "entity", units: "signals", run: func(t *testing.T, ctx context.Context, client *durabletaskscheduler.Client) int { + {name: "entity", units: "signals", serial: true, run: func(t *testing.T, ctx context.Context, client *durabletaskscheduler.Client) int { signals := scaled(1000) entityID := stressEntityID("perf-"+uuid.NewString(), 0) for range signals { @@ -849,6 +861,11 @@ func runPerf(t *testing.T, workload perfWorkload, sessions bool) perfResult { slices.Sort(latencies) if sessions { require.Positive(t, probe.acks.Load(), "the sessions-on run did not use sessions") + if workload.serial { + turns := probe.acks.Load() + probe.unary.Load() + require.LessOrEqual(t, probe.workItems.Load(), int64(2), "a serial workload loaded its instance more than twice") + require.GreaterOrEqual(t, probe.resumes.Load()*10, turns*9, "fewer than 90%% of %d turns resumed a session", turns) + } } else { require.Zero(t, probe.acks.Load(), "the sessions-off run used sessions") } @@ -857,8 +874,8 @@ func runPerf(t *testing.T, workload perfWorkload, sessions bool) perfResult { units: units, cpu: cpu, totalAlloc: after.TotalAlloc - before.TotalAlloc, - peakHeap: peakHeap.Load(), - historyBytes: probe.historyBytes.Load(), + peakHeap: peakHeap.Load() - min(peakHeap.Load(), before.HeapInuse), + payloadBytes: probe.payloadBytes.Load(), workItems: probe.workItems.Load(), replays: probe.replays.Load(), resumes: probe.resumes.Load(), @@ -915,9 +932,9 @@ func formatPerfRows(workload perfWorkload, off, on []perfResult) string { durationRow("process CPU", func(r perfResult) time.Duration { return r.cpu }) row("allocated", float64(medianOf(off, func(r perfResult) uint64 { return r.totalAlloc })), float64(medianOf(on, func(r perfResult) uint64 { return r.totalAlloc })), mib) - row("peak heap in use", float64(medianOf(off, func(r perfResult) uint64 { return r.peakHeap })), + row("peak heap growth", float64(medianOf(off, func(r perfResult) uint64 { return r.peakHeap })), float64(medianOf(on, func(r perfResult) uint64 { return r.peakHeap })), mib) - intRow("history received", func(r perfResult) int64 { return r.historyBytes }, mib) + intRow("worker protobuf payload bytes", func(r perfResult) int64 { return r.payloadBytes }, mib) intRow("work items (loads)", func(r perfResult) int64 { return r.workItems }, count) intRow("replays", func(r perfResult) int64 { return r.replays }, count) intRow("session resumes", func(r perfResult) int64 { return r.resumes }, count) @@ -936,8 +953,9 @@ func TestDTSPerf(t *testing.T) { for _, workload := range perfWorkloads() { t.Run(workload.name, func(t *testing.T) { results := map[bool][]perfResult{} - for range repeat { - for _, sessions := range []bool{false, true} { + for i := range repeat { + // Alternate the order, so warm-up does not favor one mode. + for _, sessions := range []bool{i%2 == 1, i%2 == 0} { require.NoError(t, restartBackend(t)) result := runPerf(t, workload, sessions) t.Logf("sessions=%v wall=%v acks=%d unary=%d replays=%d resumes=%d", @@ -1087,7 +1105,7 @@ func runStress(t *testing.T, config stressConfig) { if sessionsOn { sessions = &durabletaskclient.ExtendedSessionOptions{IdleTimeout: idleTimeout} } - probe := newStressProbe(config.faultRate, idleTimeout+time.Second) + probe := newStressProbe(config.faultRate, idleTimeout) workerOptions := []durabletaskclient.TaskHubGrpcWorkerOption{ durabletaskclient.WithMaxConcurrentOrchestrationWorkItems(config.orchestrations), durabletaskclient.WithMaxConcurrentActivityWorkItems(config.activities), @@ -1113,10 +1131,10 @@ func runStress(t *testing.T, config stressConfig) { var completed atomic.Int64 done := make(chan struct{}) var background sync.WaitGroup - // waitProgress waits until a share of the instances completed. It + // waitProgress waits until the given number of instances completed. It // returns false when the run ends first. - waitProgress := func(share float64) bool { - for float64(completed.Load()) < share*float64(config.instances) { + waitProgress := func(target int64) bool { + for completed.Load() < target { select { case <-done: return false @@ -1128,7 +1146,7 @@ func runStress(t *testing.T, config stressConfig) { if config.workerRestarts > 0 { background.Go(func() { for n := 1; n <= config.workerRestarts; n++ { - if !waitProgress(float64(n) / float64(config.workerRestarts+1)) { + if !waitProgress(int64(config.instances) * int64(n) / int64(config.workerRestarts+1)) { return } h.restartWorker() @@ -1136,9 +1154,10 @@ func runStress(t *testing.T, config stressConfig) { }) } persistent := os.Getenv("DTS_STRESS_PERSISTENT") == "1" + restartAt := int64(config.instances) * 3 / 10 if config.backendRestart { background.Go(func() { - if !waitProgress(0.3) { + if !waitProgress(restartAt) { return } t.Logf("restarting the backend after %d of %d instances", completed.Load(), config.instances) @@ -1148,19 +1167,24 @@ func runStress(t *testing.T, config stressConfig) { } var problems []string if config.backendRestart && !persistent { - // All state is lost on restart. Check only that new work completes. - for completed.Load() < int64(config.instances)*3/10 { - _, err := h.client.WaitForOrchestrationCompletion(ctx, batch.ids[completed.Load()]) - if err == nil { - completed.Add(1) + // All state is lost on restart. Check the instances that finish + // before it, then check that new work completes after it. + for i := range restartAt { + metadata, err := h.client.WaitForOrchestrationCompletion(ctx, batch.ids[i], api.WithFetchPayloads(true)) + require.NoError(t, err) + var got int + if metadata.RuntimeStatus != protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED { + problems = append(problems, fmt.Sprintf("%s: status %s: %v", batch.ids[i], metadata.RuntimeStatus, metadata.FailureDetails)) + } else if err := metadata.ReadOutput(&got); err != nil || got != stressExpected(int(i)) { + problems = append(problems, fmt.Sprintf("%s: output %d, want %d (%v)", batch.ids[i], got, stressExpected(int(i)), err)) } - require.NoError(t, ctx.Err()) + completed.Add(1) } background.Wait() fresh := config fresh.instances, fresh.entities, fresh.signalsPerEntity = scaled(120), scaled(12), 5 freshBatch := scheduleStressBatch(t, ctx, h.client, fresh) - problems = verifyStressBatch(ctx, h.client, freshBatch, new(atomic.Int64)) + problems = append(problems, verifyStressBatch(ctx, h.client, freshBatch, new(atomic.Int64))...) } else { problems = verifyStressBatch(ctx, h.client, batch, &completed) } @@ -1171,7 +1195,7 @@ func runStress(t *testing.T, config stressConfig) { t.Logf("finished in %v: worker restarts=%d acks=%d unary=%d replays=%d resumes=%d releases=%d abandons=%d notFound=%d maxOpenStreams=%d", elapsed, h.restarts, probe.acks.Load(), probe.unary.Load(), probe.replays.Load(), probe.resumes.Load(), probe.releases.Load(), probe.abandons.Load(), probe.notFound.Load(), probe.maxOpen.Load()) - for kind := faultCutBeforeAck; kind < stressFaultKinds; kind++ { + for kind := faultCutBeforeSend; kind < stressFaultKinds; kind++ { if n := probe.faults[kind].Load(); n > 0 { t.Logf("injected %q: %d", stressFaultNames[kind], n) } @@ -1210,6 +1234,23 @@ func runStress(t *testing.T, config stressConfig) { assert.Zero(t, probe.abandons.Load(), "unexpected abandons") assert.Zero(t, probe.notFound.Load(), "unexpected NotFound errors") } + if config.faultRate > 0 && sessionsOn && !config.backendRestart { + for kind := faultCutBeforeSend; kind < stressFaultKinds; kind++ { + assert.Positive(t, probe.faults[kind].Load(), "no %q fault was injected", stressFaultNames[kind]) + } + // Upper bounds that faults and restarts explain: + // - Only a failure before the acknowledgement abandons a session work + // item: one abandon per cut-before-send or drop-ack fault at most. + // - A worker restart can strand at most every in-flight work item: + // one abandon each, plus a NotFound for its completion and one for + // its abandon or release. + // - Every other fault ends one stream, with at most one NotFound from + // the retry or the release that follows. + inFlight := int64(h.restarts) * int64(config.orchestrations+config.activities+config.entityConcurrency) + preAck := probe.faults[faultCutBeforeSend].Load() + probe.faults[faultDropAck].Load() + assert.LessOrEqual(t, probe.abandons.Load(), preAck+inFlight, "more abandons than faults and restarts explain") + assert.LessOrEqual(t, probe.notFound.Load(), probe.faultCount()+2*inFlight, "more NotFound errors than faults and restarts explain") + } } // TestDTSStress checks correctness under load and faults. Every subtest From f2ee933c9e7b7965dd31c60f23881d74014a2bdc Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Fri, 6 Nov 2026 19:50:20 -0800 Subject: [PATCH 13/21] Count delay faults from the acknowledgement in the stress harness The worker's idle timer starts at the acknowledgement, so measure the hold from there. Assert that every fault kind was applied only when the run had enough session turns to make a missing kind meaningful. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/durabletaskscheduler/dts_stress_test.go | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/tests/durabletaskscheduler/dts_stress_test.go b/tests/durabletaskscheduler/dts_stress_test.go index 04c4cfb2..50d5d104 100644 --- a/tests/durabletaskscheduler/dts_stress_test.go +++ b/tests/durabletaskscheduler/dts_stress_test.go @@ -99,6 +99,9 @@ type stressProbe struct { open atomic.Int64 maxOpen atomic.Int64 faults [stressFaultKinds]atomic.Int64 + // delayHolds counts held messages, including holds that a worker + // restart ended before the idle timeout. + delayHolds atomic.Int64 mu sync.Mutex rng *rand.Rand @@ -278,6 +281,7 @@ type probedSessionStream struct { batch bool broken bool delay bool + ackAt time.Time } func (s *probedSessionStream) finish() { @@ -314,17 +318,18 @@ func (s *probedSessionStream) RecvMsg(m any) error { } if s.delay { // Hold back the next message past the worker's idle timeout. The - // worker usually ends the stream first. The fault counts only when the - // hold lasted at least the idle timeout. + // worker usually ends the stream first. The fault counts only when + // the idle timeout passed since the acknowledgement, which is when + // the worker's idle timer starts. s.delay = false - start := time.Now() + s.probe.delayHolds.Add(1) var cancelled bool select { case <-time.After(s.probe.idleTimeout + time.Second): case <-s.ctx.Done(): cancelled = true } - if time.Since(start) >= s.probe.idleTimeout { + if time.Since(s.ackAt) >= s.probe.idleTimeout { s.probe.faults[faultDelayAfterAck].Add(1) } if cancelled { @@ -351,6 +356,7 @@ func (s *probedSessionStream) RecvMsg(m any) error { return s.inject() } s.probe.acks.Add(1) + s.ackAt = time.Now() s.broken = s.fault == faultCutAfterAck s.delay = s.fault == faultDelayAfterAck case response.GetNewMessages() != nil: @@ -1235,8 +1241,16 @@ func runStress(t *testing.T, config stressConfig) { assert.Zero(t, probe.notFound.Load(), "unexpected NotFound errors") } if config.faultRate > 0 && sessionsOn && !config.backendRestart { - for kind := faultCutBeforeSend; kind < stressFaultKinds; kind++ { - assert.Positive(t, probe.faults[kind].Load(), "no %q fault was injected", stressFaultNames[kind]) + // About 1 in 50 session streams gets each fault kind, so with 500 + // turns a missing kind is a real problem, not bad luck. + if probe.acks.Load() >= 500 { + for kind := faultCutBeforeSend; kind < stressFaultKinds; kind++ { + applied := probe.faults[kind].Load() + if kind == faultDelayAfterAck { + applied = probe.delayHolds.Load() + } + assert.Positive(t, applied, "no %q fault was applied", stressFaultNames[kind]) + } } // Upper bounds that faults and restarts explain: // - Only a failure before the acknowledgement abandons a session work From a17cb3e2dfbf64bbea06053ddaba9ead27d8fe44 Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Fri, 6 Nov 2026 19:59:14 -0800 Subject: [PATCH 14/21] Document that a waiting extended session keeps its orchestration slot Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 2 +- client/grpc_worker_session.go | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b1676123..76621f45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Added opt-in extended sessions to gRPC workers with `WithExtendedSessions`. A session keeps an orchestration or entity loaded between turns, so DTS sends only the new messages and the worker skips the full replay. Sessions are off by default. When the service returns `Unimplemented`, the worker completes work items without sessions until it reconnects. +- Added opt-in extended sessions to gRPC workers with `WithExtendedSessions`. A session keeps an orchestration or entity loaded between turns, so DTS sends only the new messages and the worker skips the full replay. Sessions are off by default. A waiting session keeps its orchestration slot, the same as in Durable Functions, so raise `WithMaxConcurrentOrchestrationWorkItems` for slow backends or many short orchestrations. When the service returns `Unimplemented`, the worker completes work items without sessions until it reconnects. - Added required completion-transport isolation to gRPC workers: one intake and three dedicated completion/abandon connections by default, with a configurable 1–8 completion budget and bounded generation-safe retirement. `WithWorkerCompletionConnections(0)` is invalid. Borrowed workers and client listeners require separate caller-owned channels via `WithWorkerCompletionTransports`; owning factories must return distinct connections and non-nil closers for every invocation. - Added `ResourceId` connection-string support for every DTS authentication mode and optional `AuthorityHost` configuration for SDK-created DefaultAzure, WorkloadIdentity, Environment, and InteractiveBrowser credentials. Caller-supplied credentials own their authority; managed identity and developer-tool cloud configuration remain separate. - Added a flat catalogue of directly runnable DTS samples with outcome assertions, owned-resource cleanup, and process-level E2E validation selected by sample name using Go's `-run` flag. diff --git a/client/grpc_worker_session.go b/client/grpc_worker_session.go index b5ef144b..674e3407 100644 --- a/client/grpc_worker_session.go +++ b/client/grpc_worker_session.go @@ -33,8 +33,10 @@ var errSessionCleanupFailed = errors.New("extended-session cleanup failed; recon // the replay of the full history. // // A waiting session keeps its concurrency slot, a service credit, and an -// open stream. Leave sessions off for hosts that scale to zero and behind -// proxies that close idle streams quickly. +// open stream, the same as in Durable Functions. With a slow backend or many +// short orchestrations, waiting sessions can leave fewer slots for new work, +// so raise [WithMaxConcurrentOrchestrationWorkItems]. Leave sessions off for +// hosts that scale to zero and behind proxies that close idle streams quickly. type ExtendedSessionOptions struct { // IdleTimeout is the longest time a session waits for new messages after // the service accepts a turn. Zero means [DefaultExtendedSessionTimeout]. From 0c344d708031bdfe09789151305c56bdeb749245 Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Sat, 7 Nov 2026 08:46:29 -0800 Subject: [PATCH 15/21] Fix a timer race in the graceful-stop shutdown-deadline test The shutdown deadline equaled the 1 second RPC timeout of the blocked abandon. Under synctest both timers fire at the same instant, so the run could end first and Shutdown returned nil. This also fails on main. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- client/grpc_worker_drain_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/client/grpc_worker_drain_test.go b/client/grpc_worker_drain_test.go index c59fa49c..01145277 100644 --- a/client/grpc_worker_drain_test.go +++ b/client/grpc_worker_drain_test.go @@ -333,7 +333,9 @@ func TestWorkerGracefulStopKeepsRejectedItemLeaseUntilAbandoned(t *testing.T) { require.True(t, worker.Running()) if stop == "shutdown deadline" { - shutdownCtx, stopShutdown := context.WithTimeout(context.Background(), time.Second) + // Stay below the 1 s RPC timeout of the blocked abandon. With equal + // timeouts, synctest fires both at once, and the run can end first. + shutdownCtx, stopShutdown := context.WithTimeout(context.Background(), 500*time.Millisecond) defer stopShutdown() require.ErrorIs(t, worker.Shutdown(shutdownCtx), context.DeadlineExceeded) } else { From 90bab648ed8c20674447f1887f4001a4efce1d1f Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Sat, 7 Nov 2026 09:24:22 -0800 Subject: [PATCH 16/21] Bound session stream opens and batches, and fix review findings - Abandon instead of a unary fallback when the stream budget is full after an uncertain attempt. - Cover opening the session stream with the acknowledgement timeout and the end of the intake generation. - Apply the streamed-history limits to whole session batches. - Keep room for the completion count in the response size check. - Cache the entity state before externalization. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- client/grpc_worker_processor.go | 49 ++++++--- client/grpc_worker_session.go | 44 +++++++- client/grpc_worker_session_test.go | 171 ++++++++++++++++++++++++++++- 3 files changed, 240 insertions(+), 24 deletions(-) diff --git a/client/grpc_worker_processor.go b/client/grpc_worker_processor.go index fb704b6e..5c3f26ea 100644 --- a/client/grpc_worker_processor.go +++ b/client/grpc_worker_processor.go @@ -397,7 +397,7 @@ func (w *TaskHubGrpcWorker) orchestratorResponse( abandon() return nil, false } - return failOversizedOrchestratorResponse(response, w.options.maxOrchestratorCompletionBytes), true + return failOversizedOrchestratorResponse(response, w.orchestratorCompletionLimit()), true } // completeOrchestration sends a turn with the unary RPC. The count and token @@ -507,6 +507,15 @@ func (w *TaskHubGrpcWorker) runOrchestrationSession( } } +// orchestratorCompletionLimit is the size bound for a response before its +// completion count is set. Sessions send a nonzero count, so they keep room. +func (w *TaskHubGrpcWorker) orchestratorCompletionLimit() int { + if w.options.extendedSessions != nil { + return w.options.maxOrchestratorCompletionBytes - sessionCompletionCountReserve + } + return w.options.maxOrchestratorCompletionBytes +} + // canContinueOrchestrationSession reports whether a response may keep the // session. A worker-built failure, for example for an oversized response, // ends it. @@ -800,64 +809,65 @@ func (w *TaskHubGrpcWorker) processEntityWorkItem( completionToken string, request *protos.EntityRequest, ) { - result, ok := w.entityBatchResult(ctx, completionToken, request, func() { + result, state, ok := w.entityBatchResult(ctx, completionToken, request, func() { w.abandonEntity(ctx, client, completionToken) }) if !ok { return } if result.FailureDetails == nil && scope.available() { - w.runEntitySession(ctx, client, scope, request.GetInstanceId(), completionToken, result) + w.runEntitySession(ctx, client, scope, request.GetInstanceId(), completionToken, result, state) return } w.completeEntityBatchResult(ctx, client, nil, request.GetInstanceId(), result, 0) } -// entityBatchResult runs one entity batch and returns the result to send. -// It returns false after it called abandon. +// entityBatchResult runs one entity batch and returns the result to send and +// the entity state before externalization, which a session caches. It returns +// false after it called abandon. func (w *TaskHubGrpcWorker) entityBatchResult( ctx context.Context, completionToken string, request *protos.EntityRequest, abandon func(), -) (*protos.EntityBatchResult, bool) { +) (*protos.EntityBatchResult, *wrapperspb.StringValue, bool) { batch, operationInfos, err := entityBatchFromRequestV2(request) if err != nil { w.logger.Errorf("invalid V2 entity work item: %v", err) abandon() - return nil, false + return nil, nil, false } entityID, parseErr := api.EntityIDFromString(batch.GetInstanceId()) if parseErr != nil { w.logger.Errorf("%s: invalid entity instance ID: %v", batch.GetInstanceId(), parseErr) abandon() - return nil, false + return nil, nil, false } if !matchesEntityWorkItemFilters(w.options.workItemFilters, entityID.Name) { w.logger.Warnf("%s: entity work item does not match configured filters; abandoning it", request.GetInstanceId()) abandon() - return nil, false + return nil, nil, false } if err := largepayload.TransformEntityBatchRequest(ctx, w.options.largePayloads, batch); err != nil { w.logger.Errorf("%s: failed to hydrate entity batch payloads: %v", request.GetInstanceId(), err) if entityProcessingCanceled(ctx, err) { abandon() - return nil, false + return nil, nil, false } - return newEntityBatchFailure(completionToken, err), true + return newEntityBatchFailure(completionToken, err), nil, true } executor, ok := w.executor.(task.EntityExecutor) if !ok { w.logger.Error("task executor does not support entity work items") abandon() - return nil, false + return nil, nil, false } result, err := executeEntitySafely(ctx, executor, batch) if err != nil { if entityProcessingCanceled(ctx, err) { w.logger.Warnf("%s: entity execution canceled; abandoning work item", request.GetInstanceId()) abandon() - return nil, false + return nil, nil, false } w.logger.Errorf("%s: entity execution failed: %v", request.GetInstanceId(), err) result = newEntityBatchFailure(completionToken, err) @@ -871,15 +881,17 @@ func (w *TaskHubGrpcWorker) entityBatchResult( w.logger.Errorf("%s: invalid entity executor result: %v", request.GetInstanceId(), err) result = newEntityBatchFailure(completionToken, err) } + // Externalizing replaces the state field, so this keeps the full value. + state := result.EntityState if err := largepayload.TransformEntityBatchResult(ctx, w.options.largePayloads, result); err != nil { w.logger.Errorf("%s: failed to externalize entity batch payloads: %v", request.GetInstanceId(), err) if entityProcessingCanceled(ctx, err) { abandon() - return nil, false + return nil, nil, false } - return newEntityBatchFailure(completionToken, err), true + return newEntityBatchFailure(completionToken, err), nil, true } - return result, true + return result, state, true } // runEntitySession keeps an entity's state cached while the service sends new @@ -893,6 +905,7 @@ func (w *TaskHubGrpcWorker) runEntitySession( instanceID string, completionToken string, result *protos.EntityBatchResult, + resultState *wrapperspb.StringValue, ) { lease := scope.run.sessions.acquire(entitySessionKind) defer lease.release() @@ -914,7 +927,7 @@ func (w *TaskHubGrpcWorker) runEntitySession( return client.CompleteEntityTaskWithExtendedSession(callCtx, result) }, onAck: func(*protos.ExtendedSessionAck) { - state = result.EntityState + state = resultState turnCount++ }, release: releaseEntityOnce(client, completionToken), @@ -928,7 +941,7 @@ func (w *TaskHubGrpcWorker) runEntitySession( return } var ok bool - result, ok = w.entityBatchResult(ctx, completionToken, &protos.EntityRequest{ + result, resultState, ok = w.entityBatchResult(ctx, completionToken, &protos.EntityRequest{ InstanceId: instanceID, EntityState: state, OperationRequests: entityOperationEvents(batch), diff --git a/client/grpc_worker_session.go b/client/grpc_worker_session.go index 674e3407..0e9297af 100644 --- a/client/grpc_worker_session.go +++ b/client/grpc_worker_session.go @@ -12,8 +12,13 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" ) +// sessionCompletionCountReserve is the largest encoding of a non-negative +// completionCount field: a one-byte tag and a varint of up to five bytes. +const sessionCompletionCountReserve = 6 + // DefaultExtendedSessionTimeout is the default idle and transfer timeout of an // extended session. const DefaultExtendedSessionTimeout = 30 * time.Second @@ -235,8 +240,15 @@ func (w *TaskHubGrpcWorker) awaitSessionBatch( return outcome, batch } switch { - case errors.Is(preAckErr, errSessionStreamBudgetFull): + case errors.Is(preAckErr, errSessionStreamBudgetFull) && attempt == 1: return sessionUseUnary, nil + case errors.Is(preAckErr, errSessionStreamBudgetFull): + // An earlier attempt may have been applied without its + // acknowledgement. A unary completion with the same count could + // then fail and leave the session open, so abandon instead. + w.logger.Warnf("%s: no session stream is free to retry the completion; abandoning the work item", call.instanceID) + w.cleanupSessionWorkItem(ctx, scope, call.instanceID, "abandon", call.abandon) + return sessionEnded, nil case status.Code(preAckErr) == codes.Unimplemented: w.logger.Infof("%s: the service does not support extended sessions; completing without them until the next reconnect", call.instanceID) scope.connection.sessionsUnsupported.Store(true) @@ -266,7 +278,18 @@ func (w *TaskHubGrpcWorker) runSessionStream( ) (sessionOutcome, []*protos.HistoryEvent, error) { streamCtx, cancel := context.WithCancel(ctx) defer cancel() + // Opening the stream also sends the request, which can block. The + // acknowledgement timeout covers it, and so does the end of the intake + // generation. + opened := time.Now() + openTimer := time.AfterFunc(w.options.rpcTimeout, cancel) + stopGenerationLink := context.AfterFunc(scope.connection.streamCtx, cancel) stream, err := call.open(streamCtx) + openTimedOut := !openTimer.Stop() + stopGenerationLink() + if openTimedOut { + return 0, nil, status.Error(codes.DeadlineExceeded, "opening the session stream timed out") + } if err != nil { return 0, nil, err } @@ -291,7 +314,7 @@ func (w *TaskHubGrpcWorker) runSessionStream( <-received }() - timer, stopTimer := w.sessionTimer(sessionAckTimer, w.options.rpcTimeout) + timer, stopTimer := w.sessionTimer(sessionAckTimer, max(0, w.options.rpcTimeout-time.Since(opened))) defer func() { stopTimer() }() restartTimer := func(kind sessionTimerKind, timeout time.Duration) { stopTimer() @@ -307,7 +330,10 @@ func (w *TaskHubGrpcWorker) runSessionStream( } acked, transferring := false, false - var batch []*protos.HistoryEvent + var ( + batch []*protos.HistoryEvent + batchBytes int64 + ) for { // Shutdown ends a session only while it waits. var stopping <-chan struct{} @@ -363,7 +389,17 @@ func (w *TaskHubGrpcWorker) runSessionStream( transferring = true restartTimer(sessionTransferTimer, w.options.extendedSessions.TransferTimeout) } - batch = append(batch, response.NewMessages.GetEvents()...) + // Apply the streamed-history limits to the whole batch. + events := response.NewMessages.GetEvents() + if limit := w.options.maxStreamedHistoryEvents; limit > 0 && len(events) > limit-len(batch) { + return end(fmt.Sprintf("the batch exceeds %d events", limit), true) + } + chunkBytes := int64(proto.Size(response.NewMessages)) + if limit := w.options.maxStreamedHistoryBytes; limit > 0 && chunkBytes > limit-batchBytes { + return end(fmt.Sprintf("the batch exceeds %d bytes", limit), true) + } + batch = append(batch, events...) + batchBytes += chunkBytes default: if !acked { return 0, nil, status.Error(codes.Internal, "the service sent an unknown session message") diff --git a/client/grpc_worker_session_test.go b/client/grpc_worker_session_test.go index ba569530..cdcda471 100644 --- a/client/grpc_worker_session_test.go +++ b/client/grpc_worker_session_test.go @@ -14,6 +14,7 @@ import ( "github.com/microsoft/durabletask-go/api" "github.com/microsoft/durabletask-go/internal/helpers" "github.com/microsoft/durabletask-go/internal/protos" + "github.com/microsoft/durabletask-go/payload" "github.com/microsoft/durabletask-go/task" "github.com/stretchr/testify/require" "google.golang.org/grpc" @@ -86,8 +87,10 @@ func (c *fakeSessionCall) count() int32 { // sessionSchedulerClient adds the extended-session RPCs to the fake scheduler. type sessionSchedulerClient struct { *fakeSchedulerClient - calls chan *fakeSessionCall - openErr func() error + calls chan *fakeSessionCall + openErr func() error + // openWait runs before the stream opens and can block like a send. + openWait func(context.Context) error unaryErr func() error releaseErr func() error abandonErr func() error @@ -127,6 +130,11 @@ func (c *sessionSchedulerClient) CompleteOrchestratorTask( } func (c *sessionSchedulerClient) open(ctx context.Context, call *fakeSessionCall) (grpc.ServerStreamingClient[protos.CompleteTaskWithExtendedSessionResponse], error) { + if c.openWait != nil { + if err := c.openWait(ctx); err != nil { + return nil, err + } + } if c.openErr != nil { if err := c.openErr(); err != nil { return nil, err @@ -1009,3 +1017,162 @@ func (*erroringStreamConn) NewStream(context.Context, *grpc.StreamDesc, string, type erroringClientStream struct{ grpc.ClientStream } func (erroringClientStream) RecvMsg(any) error { return io.EOF } + +func TestWorkerSessionBudgetFullAfterUncertainAttemptAbandons(t *testing.T) { + h := newSessionHarness(t, 2) + var opens atomic.Int32 + h.client.openErr = func() error { + if opens.Add(1) == 2 { + return errSessionStreamBudgetFull + } + return nil + } + h.dispatch("uncertain") + // The service may have applied this attempt before the stream broke. + h.client.nextCall(t).stream.end(status.Error(codes.Unavailable, "lost")) + require.Eventually(t, func() bool { return h.client.abandons() == 1 }, 5*time.Second, time.Millisecond) + require.EqualValues(t, 2, opens.Load()) + h.client.noCall(t) + require.Empty(t, h.client.unaryOrchestrations(), "a unary completion after an uncertain attempt can leave the session open") +} + +func TestWorkerSessionOpenTimeoutRetries(t *testing.T) { + h := newSessionHarness(t, 1, WithWorkerRPCTimeout(200*time.Millisecond)) + var opens atomic.Int32 + h.client.openWait = func(ctx context.Context) error { + if opens.Add(1) == 1 { + <-ctx.Done() + return status.FromContextError(ctx.Err()).Err() + } + return nil + } + h.dispatch("slow-open") + retried := h.client.nextCall(t) + require.EqualValues(t, 2, opens.Load(), "a stuck open must time out and retry") + require.EqualValues(t, 0, retried.count()) + require.Zero(t, h.client.abandons()) +} + +func TestWorkerSessionOpenStopsWhenIntakeEnds(t *testing.T) { + h := newSessionHarness(t, 1, WithWorkerRPCTimeout(time.Minute)) + opening := make(chan struct{}) + unblocked := make(chan struct{}) + var opens atomic.Int32 + h.client.openWait = func(ctx context.Context) error { + if opens.Add(1) > 1 { + return nil + } + close(opening) + <-ctx.Done() + close(unblocked) + return status.FromContextError(ctx.Err()).Err() + } + h.dispatch("stuck") + <-opening + h.client.stream.results <- fakeWorkItemResult{err: status.Error(codes.Unavailable, "intake ended")} + select { + case <-unblocked: + case <-time.After(5 * time.Second): + t.Fatal("the stuck open did not stop when its intake generation ended") + } + require.Never(t, func() bool { + return h.client.abandons() != 0 || len(h.client.unaryOrchestrations()) != 0 + }, 50*time.Millisecond, 5*time.Millisecond) +} + +func TestWorkerSessionBatchLimitsRelease(t *testing.T) { + t.Run("events", func(t *testing.T) { + h := newSessionHarness(t, 2, WithMaxStreamedHistoryEvents(2)) + h.dispatch("events") + first := h.client.nextCall(t) + first.stream.ack(4) + first.stream.chunk(helpers.NewOrchestratorStartedEvent(), helpers.NewEventRaisedEvent("x", nil)) + first.stream.chunk(helpers.NewTaskCompletedEvent(0, nil)) + require.Eventually(t, func() bool { return h.client.releases.Load() == 1 }, 5*time.Second, time.Millisecond) + h.client.noCall(t) + require.Empty(t, h.client.unaryOrchestrations()) + }) + t.Run("bytes", func(t *testing.T) { + h := newSessionHarness(t, 2, WithMaxStreamedHistoryBytes(256)) + h.dispatch("bytes") + first := h.client.nextCall(t) + first.stream.ack(4) + first.stream.chunk(helpers.NewOrchestratorStartedEvent(), + helpers.NewEventRaisedEvent("x", wrapperspb.String(strings.Repeat("x", 512)))) + require.Eventually(t, func() bool { return h.client.releases.Load() == 1 }, 5*time.Second, time.Millisecond) + h.client.noCall(t) + require.Empty(t, h.client.unaryOrchestrations()) + }) +} + +func TestWorkerSessionReservesCompletionCountBytes(t *testing.T) { + for _, message := range []proto.Message{&protos.OrchestratorResponse{InstanceId: "x"}, &protos.EntityBatchResult{}} { + before := proto.Size(message) + switch m := message.(type) { + case *protos.OrchestratorResponse: + m.CompletionCount = math.MaxInt32 + case *protos.EntityBatchResult: + m.CompletionCount = math.MaxInt32 + } + require.LessOrEqual(t, proto.Size(message)-before, sessionCompletionCountReserve) + } + off := newSessionHarness(t, 1, WithMaxOrchestratorCompletionBytes(64*1024)).worker + off.options.extendedSessions = nil + require.Equal(t, 64*1024, off.orchestratorCompletionLimit()) + on := newSessionHarness(t, 1, WithMaxOrchestratorCompletionBytes(64*1024)).worker + require.Equal(t, 64*1024-sessionCompletionCountReserve, on.orchestratorCompletionLimit()) +} + +type countingPayloadStore struct { + *payload.MemoryStore + resolves atomic.Int32 +} + +func (s *countingPayloadStore) Resolve(ctx context.Context, location string) ([]byte, error) { + s.resolves.Add(1) + return s.MemoryStore.Resolve(ctx, location) +} + +func TestWorkerEntitySessionCachesStateBeforeExternalization(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddEntityN("text", func(ctx *task.EntityContext) (any, error) { + var value string + if ctx.HasState() { + if err := ctx.GetState(&value); err != nil { + return nil, err + } + } + return nil, ctx.SetState(value + "x") + })) + store := &countingPayloadStore{MemoryStore: payload.NewMemoryStore()} + client := newSessionSchedulerClient() + h := startSessionWorker(t, registry, client, WithWorkerLargePayloads(&api.LargePayloadOptions{ + Store: store, Resolver: store, ThresholdBytes: 1, MaxPayloadBytes: 1024, + })) + var requests []*protos.EntityBatchRequest + var mu sync.Mutex + base := h.worker.executor.(task.EntityExecutor) + h.worker.executor = &recordingExecutor{executeEntity: func(ctx context.Context, request *protos.EntityBatchRequest) (*protos.EntityBatchResult, error) { + mu.Lock() + requests = append(requests, proto.Clone(request).(*protos.EntityBatchRequest)) + mu.Unlock() + return base.ExecuteEntity(ctx, request) + }} + client.stream.results <- fakeWorkItemResult{item: &protos.WorkItem{ + Request: &protos.WorkItem_EntityRequestV2{EntityRequestV2: entitySessionRequest("@text@a", entitySignal("add"))}, + CompletionToken: "entity-token", + }} + + first := client.nextCall(t) + require.True(t, api.IsLargePayloadReference(first.entity.EntityState.GetValue()), "the service must get the externalized state") + first.stream.ack(0) + first.stream.batch(helpers.NewOrchestratorStartedEvent(), entitySignal("add")) + + second := client.nextCall(t) + require.True(t, api.IsLargePayloadReference(second.entity.EntityState.GetValue())) + require.Zero(t, store.resolves.Load(), "a resumed batch must not download the cached state") + mu.Lock() + defer mu.Unlock() + require.Len(t, requests, 2) + require.Equal(t, `"x"`, requests[1].EntityState.GetValue()) +} From 97de25cea0c8403c4d27e3952817fdcfeede72b4 Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Sat, 7 Nov 2026 09:24:22 -0800 Subject: [PATCH 17/21] Drop finished coroutines and sort pending action IDs between session turns Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- task/orchestrator.go | 14 ++++++----- task/scheduler.go | 18 ++++++++++++++ task/session_test.go | 58 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 6 deletions(-) diff --git a/task/orchestrator.go b/task/orchestrator.go index 0fbc7294..a1427cd9 100644 --- a/task/orchestrator.go +++ b/task/orchestrator.go @@ -8,6 +8,7 @@ import ( "maps" "math" "runtime/debug" + "slices" "sort" "strings" "time" @@ -434,6 +435,7 @@ func (ctx *OrchestrationContext) releaseProcessedHistory() { ctx.oldEvents = nil ctx.newEvents = nil ctx.historyIndex = 0 + ctx.scheduler.dropCompleted() } // resume runs one more turn on the loaded orchestration. The batch contains @@ -1891,12 +1893,12 @@ func (ctx *OrchestrationContext) actions() []*protos.OrchestratorAction { return nil } - actions := make([]*protos.OrchestratorAction, 0, len(ctx.pendingActions)) - for id := int32(0); id < ctx.sequenceNumber; id++ { - a, ok := ctx.pendingActions[id] - if !ok { - continue - } + // Sort the pending IDs instead of scanning every ID ever used, because a + // long session retires most of them. + ids := slices.Sorted(maps.Keys(ctx.pendingActions)) + actions := make([]*protos.OrchestratorAction, 0, len(ids)) + for _, id := range ids { + a := ctx.pendingActions[id] actions = append(actions, a) // ContinueAsNew intent survives a later failure or termination, so check // the completion's actual status before carrying events forward. diff --git a/task/scheduler.go b/task/scheduler.go index 29113840..d854d743 100644 --- a/task/scheduler.go +++ b/task/scheduler.go @@ -3,6 +3,7 @@ package task import ( "container/heap" "fmt" + "slices" ) type coroutineQueue []*coroutine @@ -177,6 +178,23 @@ func (s *coroutineScheduler) isStopping() bool { return s.stopping } +// dropCompleted forgets finished coroutines between session turns, so a long +// session does not keep their closures. Cancellation and shutdown skip +// finished coroutines, so this changes no behavior. +func (s *coroutineScheduler) dropCompleted() { + finished := func(c *coroutine) bool { return c.state == coroutineCompleted && c != s.root } + scopes := make(map[*cancellationScope]struct{}) + for _, c := range s.all { + if finished(c) && c.scope != nil { + scopes[c.scope] = struct{}{} + } + } + for scope := range scopes { + scope.coroutines = slices.DeleteFunc(scope.coroutines, finished) + } + s.all = slices.DeleteFunc(s.all, finished) +} + func (s *coroutineScheduler) shutdown() { if s.stopping { return diff --git a/task/session_test.go b/task/session_test.go index 9402d3ff..2cfb8ef2 100644 --- a/task/session_test.go +++ b/task/session_test.go @@ -1105,3 +1105,61 @@ func TestSessionCloseEmitsNoDurableWork(t *testing.T) { require.Equal(t, 3, deferredBlocked) requireNoGoroutineLeak(t, baseline) } + +func TestSessionDropsFinishedCoroutinesBetweenTurns(t *testing.T) { + ctx, session, response := startWaitingSession(t, func(ctx *OrchestrationContext) (any, error) { + childDone := false + ctx.Go(func(child *OrchestrationContext) { + if child.WaitForSingleEvent("child", -1).Await(nil) == nil { + childDone = true + } + }) + for range 3 { + if err := ctx.WaitForSingleEvent("e", -1).Await(nil); err != nil { + return nil, err + } + ctx.Go(func(*OrchestrationContext) {}) + } + if err := ctx.WaitForSingleEvent("end", -1).Await(nil); err != nil { + return nil, err + } + return childDone, nil + }) + live := func(want int) { + t.Helper() + require.Len(t, ctx.scheduler.all, want) + for _, c := range ctx.scheduler.all { + if c.scope != nil { + for _, scoped := range c.scope.coroutines { + require.NotEqual(t, coroutineCompleted, scoped.state, "a finished coroutine is still in its scope") + } + } + } + } + live(2) + actions := response.Actions + historyBase := 2 + turn := func(event string) (*ExecutionResults, OrchestratorSession) { + t.Helper() + session.Commit(actions) + historyBase += 2 + result, next, err := session.Resume(context.Background(), []*protos.HistoryEvent{ + sessionOrchestratorStarted(sessionTestStart.Add(time.Duration(historyBase) * time.Minute)), + eventRaised(event, `1`), + }, historyBase) + require.NoError(t, err) + actions = result.Response.Actions + return result, next + } + for range 3 { + _, next := turn("e") + require.NotNil(t, next) + live(2) + } + _, next := turn("child") + require.NotNil(t, next) + live(1) + result, next := turn("end") + require.Nil(t, next) + requireCompletedWith(t, sessionRun{responses: []*protos.OrchestratorResponse{result.Response}, sessionEnded: true}, `true`) +} From 5e3380f084913ce452f205e220b841f64b75e6f4 Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Sat, 7 Nov 2026 09:24:22 -0800 Subject: [PATCH 18/21] Make stress harness cleanup and gates safe on failures and low fault rates Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/durabletaskscheduler/dts_stress_test.go | 34 ++++++++++++++----- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/tests/durabletaskscheduler/dts_stress_test.go b/tests/durabletaskscheduler/dts_stress_test.go index 50d5d104..f22056d8 100644 --- a/tests/durabletaskscheduler/dts_stress_test.go +++ b/tests/durabletaskscheduler/dts_stress_test.go @@ -851,13 +851,21 @@ func runPerf(t *testing.T, workload perfWorkload, sessions bool) perfResult { } } }() + var stopOnce sync.Once + stop := func() { + stopOnce.Do(func() { + close(stopSampler) + <-sampled + }) + } + // A failed workload exits through runtime.Goexit, so stop the sampler then too. + defer stop() cpuStart := processCPU() start := time.Now() units := workload.run(t, ctx, h.client) wall := time.Since(start) cpu := processCPU() - cpuStart - close(stopSampler) - <-sampled + stop() var after runtime.MemStats runtime.ReadMemStats(&after) @@ -868,9 +876,12 @@ func runPerf(t *testing.T, workload perfWorkload, sessions bool) perfResult { if sessions { require.Positive(t, probe.acks.Load(), "the sessions-on run did not use sessions") if workload.serial { + // DTS may combine entity signals into a few batches, so check that + // every turn after the first two loads resumed a session, not a + // share of turns. turns := probe.acks.Load() + probe.unary.Load() require.LessOrEqual(t, probe.workItems.Load(), int64(2), "a serial workload loaded its instance more than twice") - require.GreaterOrEqual(t, probe.resumes.Load()*10, turns*9, "fewer than 90%% of %d turns resumed a session", turns) + require.GreaterOrEqual(t, probe.resumes.Load(), turns-2, "only %d of %d turns resumed a session", probe.resumes.Load(), turns) } } else { require.Zero(t, probe.acks.Load(), "the sessions-off run used sessions") @@ -1137,6 +1148,13 @@ func runStress(t *testing.T, config stressConfig) { var completed atomic.Int64 done := make(chan struct{}) var background sync.WaitGroup + var stopOnce sync.Once + stopBackground := func() { + stopOnce.Do(func() { close(done) }) + background.Wait() + } + // A failed check exits through runtime.Goexit, so stop the restarts then too. + defer stopBackground() // waitProgress waits until the given number of instances completed. It // returns false when the run ends first. waitProgress := func(target int64) bool { @@ -1194,8 +1212,7 @@ func runStress(t *testing.T, config stressConfig) { } else { problems = verifyStressBatch(ctx, h.client, batch, &completed) } - close(done) - background.Wait() + stopBackground() elapsed := time.Since(start) t.Logf("finished in %v: worker restarts=%d acks=%d unary=%d replays=%d resumes=%d releases=%d abandons=%d notFound=%d maxOpenStreams=%d", @@ -1241,9 +1258,10 @@ func runStress(t *testing.T, config stressConfig) { assert.Zero(t, probe.notFound.Load(), "unexpected NotFound errors") } if config.faultRate > 0 && sessionsOn && !config.backendRestart { - // About 1 in 50 session streams gets each fault kind, so with 500 - // turns a missing kind is a real problem, not bad luck. - if probe.acks.Load() >= 500 { + // Each stream gets each fault kind with a chance of faultRate/5. With + // 10 expected faults of each kind, a missing kind is a real problem, + // not bad luck (the default rate of 0.1 needs 500 turns). + if float64(probe.acks.Load())*config.faultRate/float64(stressFaultKinds-1) >= 10 { for kind := faultCutBeforeSend; kind < stressFaultKinds; kind++ { applied := probe.faults[kind].Load() if kind == faultDelayAfterAck { From b9ceaf7edd29d8cfe1631a436a68bd890e26c12a Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Sat, 7 Nov 2026 09:53:41 -0800 Subject: [PATCH 19/21] End a session after orchestrator code reads turn-wide history values A full replay reads HistoryLength and HistoryLimitExceeded again with a later turn's value, so a session that keeps running could decide differently. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- task/executor.go | 4 ++-- task/orchestrator.go | 56 +++++++++++++++++++++++++++----------------- task/session.go | 5 +++- task/session_test.go | 34 +++++++++++++++++++++++++++ 4 files changed, 74 insertions(+), 25 deletions(-) diff --git a/task/executor.go b/task/executor.go index 903e148c..899563b6 100644 --- a/task/executor.go +++ b/task/executor.go @@ -588,9 +588,9 @@ func (te *taskExecutor) reportHistoryMetric(ctx *OrchestrationContext) { InstanceID: ctx.ID, OrchestrationName: ctx.Name, OrchestrationVersion: ctx.Version, - HistoryLength: ctx.HistoryLength(), + HistoryLength: ctx.historyLength(), ProcessedEvents: ctx.processedEventsThisTurn, - HistoryLimitExceeded: ctx.HistoryLimitExceeded(), + HistoryLimitExceeded: ctx.maxHistoryEventsExceeded, } defer func() { if recovered := recover(); recovered != nil { diff --git a/task/orchestrator.go b/task/orchestrator.go index a1427cd9..e1d11317 100644 --- a/task/orchestrator.go +++ b/task/orchestrator.go @@ -72,23 +72,26 @@ type OrchestrationContext struct { processedEventsThisTurn int maxEventsPerTurnExceeded bool maxHistoryEventsExceeded bool - sequenceNumber int32 - newGuidCounter uint64 - pendingActions map[int32]*protos.OrchestratorAction - pendingTasks map[int32]*completableTask - pendingEntityTasks map[string]*completableTask - continuedAsNew bool - continuedAsNewInput any - continuedAsNewVersion *wrapperspb.StringValue - customStatus string - defaultVersion string - converter api.DataConverter - entitiesSupported bool - scheduler *coroutineScheduler - unwinding bool - root *OrchestrationContext - scope *cancellationScope - derived []*OrchestrationContext + // turnHistoryRead records that orchestrator code read HistoryLength or + // HistoryLimitExceeded, which an extended session cannot keep consistent. + turnHistoryRead bool + sequenceNumber int32 + newGuidCounter uint64 + pendingActions map[int32]*protos.OrchestratorAction + pendingTasks map[int32]*completableTask + pendingEntityTasks map[string]*completableTask + continuedAsNew bool + continuedAsNewInput any + continuedAsNewVersion *wrapperspb.StringValue + customStatus string + defaultVersion string + converter api.DataConverter + entitiesSupported bool + scheduler *coroutineScheduler + unwinding bool + root *OrchestrationContext + scope *cancellationScope + derived []*OrchestrationContext bufferedExternalEvents map[string]*list.List pendingExternalEventTasks map[string]*list.List @@ -431,7 +434,7 @@ func (ctx *OrchestrationContext) commit(sent []*protos.OrchestratorAction) { // buffers keep everything that later turns need, so a long session does not // keep its full history in memory. func (ctx *OrchestrationContext) releaseProcessedHistory() { - ctx.historyBase = ctx.HistoryLength() + ctx.historyBase = ctx.historyLength() ctx.oldEvents = nil ctx.newEvents = nil ctx.historyIndex = 0 @@ -602,17 +605,26 @@ func countsTowardTurnBudget(event *protos.HistoryEvent) bool { } // HistoryLength returns the total old and new history of this execution turn. +// A full replay reads it again with the value of a later turn, so with +// extended sessions, reading it ends the session after the current turn. func (ctx *OrchestrationContext) HistoryLength() int { engine := ctx.engineContext() - return engine.historyBase + len(engine.newEvents) + engine.turnHistoryRead = true + return engine.historyLength() } -// HistoryLimitExceeded reports whether MaxHistoryEvents was exceeded. +// HistoryLimitExceeded reports whether MaxHistoryEvents was exceeded. Like +// HistoryLength, reading it ends an extended session after the current turn. func (ctx *OrchestrationContext) HistoryLimitExceeded() bool { engine := ctx.engineContext() + engine.turnHistoryRead = true return engine.maxHistoryEventsExceeded } +func (ctx *OrchestrationContext) historyLength() int { + return ctx.historyBase + len(ctx.newEvents) +} + func (ctx *OrchestrationContext) processEvent(e *protos.HistoryEvent) error { defer ctx.syncDerivedContexts() // Buffer certain events if we're in a suspended state @@ -1741,7 +1753,7 @@ func (ctx *OrchestrationContext) enforceHistoryLimit() { InstanceID: ctx.ID, OrchestrationName: ctx.Name, OrchestrationVersion: ctx.Version, - HistoryLength: ctx.HistoryLength(), + HistoryLength: ctx.historyLength(), MaxHistoryEvents: ctx.orchestrationOptions.MaxHistoryEvents, UnprocessedEventCount: unprocessedEventCount, SerializedInput: string(ctx.rawInput), @@ -1781,7 +1793,7 @@ func invokeHistoryLimitHandler(handler HistoryLimitHandler, info HistoryLimitInf func (ctx *OrchestrationContext) newHistoryLimitError(policyError error) *HistoryLimitError { return &HistoryLimitError{ InstanceID: ctx.ID, - HistoryLength: ctx.HistoryLength(), + HistoryLength: ctx.historyLength(), MaxHistoryEvents: ctx.orchestrationOptions.MaxHistoryEvents, PolicyError: policyError, } diff --git a/task/session.go b/task/session.go index a2dd6762..e1eaf69a 100644 --- a/task/session.go +++ b/task/session.go @@ -103,7 +103,10 @@ func (ctx *OrchestrationContext) canContinueSession() bool { // The backend sends unprocessed events again in the next work item, so // a partial turn must not keep them in memory. !ctx.maxEventsPerTurnExceeded && - !ctx.maxHistoryEventsExceeded + !ctx.maxHistoryEventsExceeded && + // A full replay would read these values again with a later turn's + // value, so the session could decide differently. + !ctx.turnHistoryRead } func validateSessionBatch(batch []*protos.HistoryEvent, historyBase int) error { diff --git a/task/session_test.go b/task/session_test.go index 2cfb8ef2..5a99afd4 100644 --- a/task/session_test.go +++ b/task/session_test.go @@ -1163,3 +1163,37 @@ func TestSessionDropsFinishedCoroutinesBetweenTurns(t *testing.T) { require.Nil(t, next) requireCompletedWith(t, sessionRun{responses: []*protos.OrchestratorResponse{result.Response}, sessionEnded: true}, `true`) } + +// A full replay reads HistoryLength again with the latest turn's value. The +// session must end once orchestrator code reads it, or a later turn could +// decide differently from the replay. +func TestSessionEndsAfterTurnHistoryRead(t *testing.T) { + for _, tt := range []struct { + name string + read func(*OrchestrationContext) bool + }{ + {name: "HistoryLength", read: func(ctx *OrchestrationContext) bool { return ctx.HistoryLength() > 9 }}, + {name: "HistoryLimitExceeded", read: func(ctx *OrchestrationContext) bool { return ctx.HistoryLimitExceeded() }}, + } { + t.Run(tt.name, func(t *testing.T) { + run := runSessionEquivalence(t, sessionScenario{orchestrator: func(ctx *OrchestrationContext) (any, error) { + for i := 0; ; i++ { + // The first reads happen on turn 2, so the session must + // survive the earlier turns. + if i >= 2 && tt.read(ctx) { + return i, nil + } + if i == 6 { + return i, nil + } + if err := ctx.CallActivity("work").Await(nil); err != nil { + return nil, err + } + } + }}) + require.True(t, run.sessionEnded) + require.Len(t, run.responses, 3, "the session must end on the turn that read the value") + require.Nil(t, completionOf(run.responses[2])) + }) + } +} From 9209c6eadeabde702a2fc1125ae01930978c4e25 Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Sat, 7 Nov 2026 09:53:41 -0800 Subject: [PATCH 20/21] Stop session retries and cleanup when the intake stream ends Also abandon instead of falling back to unary when a retry after an uncertain attempt returns Unimplemented. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- client/grpc_worker_session.go | 40 +++++++++++---- client/grpc_worker_session_test.go | 80 ++++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+), 9 deletions(-) diff --git a/client/grpc_worker_session.go b/client/grpc_worker_session.go index 0e9297af..1328ea6e 100644 --- a/client/grpc_worker_session.go +++ b/client/grpc_worker_session.go @@ -181,6 +181,17 @@ func sessionScopeAfterAck(scope *workerSessionScope, turnCount int32) *workerSes return scope } +// generationContext returns a context that also ends when the intake stream +// of this generation ends. +func (s *workerSessionScope) generationContext(ctx context.Context) (context.Context, context.CancelFunc) { + generationCtx, cancel := context.WithCancel(ctx) + stop := context.AfterFunc(s.connection.streamCtx, cancel) + return generationCtx, func() { + stop() + cancel() + } +} + // available reports whether a new session turn may use the streaming RPC. func (s *workerSessionScope) available() bool { return s != nil && s.run.sessions != nil && @@ -239,24 +250,28 @@ func (w *TaskHubGrpcWorker) awaitSessionBatch( if preAckErr == nil { return outcome, batch } + unimplemented := status.Code(preAckErr) == codes.Unimplemented + if unimplemented { + w.logger.Infof("%s: the service does not support extended sessions; completing without them until the next reconnect", call.instanceID) + scope.connection.sessionsUnsupported.Store(true) + } switch { - case errors.Is(preAckErr, errSessionStreamBudgetFull) && attempt == 1: + case (unimplemented || errors.Is(preAckErr, errSessionStreamBudgetFull)) && attempt == 1: return sessionUseUnary, nil - case errors.Is(preAckErr, errSessionStreamBudgetFull): + case unimplemented || errors.Is(preAckErr, errSessionStreamBudgetFull): // An earlier attempt may have been applied without its // acknowledgement. A unary completion with the same count could // then fail and leave the session open, so abandon instead. - w.logger.Warnf("%s: no session stream is free to retry the completion; abandoning the work item", call.instanceID) + w.logger.Warnf("%s: cannot retry the completion on a session stream; abandoning the work item: %v", call.instanceID, preAckErr) w.cleanupSessionWorkItem(ctx, scope, call.instanceID, "abandon", call.abandon) return sessionEnded, nil - case status.Code(preAckErr) == codes.Unimplemented: - w.logger.Infof("%s: the service does not support extended sessions; completing without them until the next reconnect", call.instanceID) - scope.connection.sessionsUnsupported.Store(true) - return sessionUseUnary, nil case ctx.Err() != nil || scope.connection.streamCtx.Err() != nil: return sessionEnded, nil case isTransientWorkerRPCError(preAckErr) && attempt < w.options.transientRetryMaxAttempts: - if w.wait(ctx, w.retryDelay(attempt)) != nil { + waitCtx, cancelWait := scope.generationContext(ctx) + err := w.wait(waitCtx, w.retryDelay(attempt)) + cancelWait() + if err != nil { return sessionEnded, nil } default: @@ -428,7 +443,11 @@ func (w *TaskHubGrpcWorker) cleanupSessionWorkItem( operation string, action func(context.Context) error, ) { - cleanupCtx, cancel := context.WithTimeout(ctx, w.options.sessionCleanupTimeout) + // When the intake stream ends, the service takes the work item back, so + // cleanup stops too. + generationCtx, cancelGeneration := scope.generationContext(ctx) + defer cancelGeneration() + cleanupCtx, cancel := context.WithTimeout(generationCtx, w.options.sessionCleanupTimeout) defer cancel() for attempt := 1; ; attempt++ { callCtx, cancelCall := context.WithTimeout(cleanupCtx, w.options.rpcTimeout) @@ -441,6 +460,9 @@ func (w *TaskHubGrpcWorker) cleanupSessionWorkItem( w.wait(cleanupCtx, w.retryDelay(attempt)) == nil && cleanupCtx.Err() == nil { continue } + if ctx.Err() != nil || scope.connection.streamCtx.Err() != nil { + return + } w.logger.Errorf("%s: failed to %s the session work item; reconnecting the work item stream: %v", instanceID, operation, err) scope.connection.abortForSessionCleanup() return diff --git a/client/grpc_worker_session_test.go b/client/grpc_worker_session_test.go index cdcda471..0ed2904b 100644 --- a/client/grpc_worker_session_test.go +++ b/client/grpc_worker_session_test.go @@ -1176,3 +1176,83 @@ func TestWorkerEntitySessionCachesStateBeforeExternalization(t *testing.T) { require.Len(t, requests, 2) require.Equal(t, `"x"`, requests[1].EntityState.GetValue()) } + +func TestWorkerSessionUnimplementedAfterUncertainAttemptAbandons(t *testing.T) { + h := newSessionHarness(t, 2) + var opens atomic.Int32 + h.client.openErr = func() error { + if opens.Add(1) == 2 { + return status.Error(codes.Unimplemented, "switched off") + } + return nil + } + h.dispatch("uncertain") + h.client.nextCall(t).stream.end(status.Error(codes.Unavailable, "lost")) + require.Eventually(t, func() bool { return h.client.abandons() == 1 }, 5*time.Second, time.Millisecond) + require.Empty(t, h.client.unaryOrchestrations(), "a unary completion after an uncertain attempt can leave the session open") + + // Sessions stay off on this intake generation. + h.dispatch("next") + require.Len(t, h.waitUnary(t, 1), 1) + h.client.noCall(t) +} + +// detachedSessionScope returns a session scope whose intake stream the test +// ends by calling the returned function. +func detachedSessionScope() (*workerSessionScope, context.CancelFunc) { + streamCtx, endStream := context.WithCancel(context.Background()) + connection := &grpcWorkerConnection{streamCtx: streamCtx, cancelStream: endStream} + run := &grpcWorkerRun{intakeCtx: context.Background()} + return &workerSessionScope{run: run, connection: connection}, endStream +} + +func TestWorkerSessionRetryWaitStopsWhenIntakeEnds(t *testing.T) { + worker := newSessionHarness(t, 1, WithWorkerTransientRetryPolicy(3, time.Hour, time.Hour)).worker + scope, endStream := detachedSessionScope() + opened := make(chan struct{}, 3) + outcome := make(chan sessionOutcome, 1) + go func() { + result, _ := worker.awaitSessionBatch(context.Background(), scope, sessionCall{ + instanceID: "retry-wait", + open: func(context.Context) (grpc.ServerStreamingClient[protos.CompleteTaskWithExtendedSessionResponse], error) { + opened <- struct{}{} + return nil, status.Error(codes.Unavailable, "down") + }, + abandon: func(context.Context) error { return nil }, + }) + outcome <- result + }() + <-opened + endStream() + select { + case result := <-outcome: + require.Equal(t, sessionEnded, result) + case <-time.After(5 * time.Second): + t.Fatal("the retry wait did not stop when the intake stream ended") + } + require.Len(t, opened, 0, "no attempt may run after the intake stream ended") +} + +func TestWorkerSessionCleanupStopsWhenIntakeEnds(t *testing.T) { + worker := newSessionHarness(t, 1, WithWorkerTransientRetryPolicy(3, time.Hour, time.Hour)).worker + worker.options.sessionCleanupTimeout = time.Hour + scope, endStream := detachedSessionScope() + attempts := make(chan struct{}, 3) + done := make(chan struct{}) + go func() { + defer close(done) + worker.cleanupSessionWorkItem(context.Background(), scope, "cleanup", "release", func(context.Context) error { + attempts <- struct{}{} + return status.Error(codes.Unavailable, "down") + }) + }() + <-attempts + endStream() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("cleanup did not stop when the intake stream ended") + } + require.Len(t, attempts, 0) + require.False(t, scope.connection.sessionCleanupFailed.Load(), "an ended generation needs no extra reconnect") +} From 1bec35e7e5c2401cfb017b5e59862cd6379d6db1 Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Sat, 7 Nov 2026 09:53:41 -0800 Subject: [PATCH 21/21] Make session E2E and stress readiness checks robust to timing Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../dts_extended_sessions_test.go | 5 ++- tests/durabletaskscheduler/dts_stress_test.go | 31 +++++++++++++------ 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/tests/durabletaskscheduler/dts_extended_sessions_test.go b/tests/durabletaskscheduler/dts_extended_sessions_test.go index f264b9c4..b8568ffb 100644 --- a/tests/durabletaskscheduler/dts_extended_sessions_test.go +++ b/tests/durabletaskscheduler/dts_extended_sessions_test.go @@ -406,6 +406,10 @@ func TestDTSExtendedSessionReleaseRace(t *testing.T) { require.NoError(t, err) _, err = client.WaitForOrchestrationStart(ctx, id) require.NoError(t, err) + // The idle timer starts at the acknowledgement, not at RaiseEvent, so wait + // for the first idle release instead of relying on the delays below. + require.Eventually(t, func() bool { return tracker.releases.Load() >= 1 }, time.Minute, 10*time.Millisecond, + "the idle timeout must release the waiting session") for i := 1; i <= 8; i++ { time.Sleep(time.Duration(40*(i%4)) * time.Millisecond) require.NoError(t, client.RaiseEvent(ctx, id, "value", api.WithEventPayload(i))) @@ -413,7 +417,6 @@ func TestDTSExtendedSessionReleaseRace(t *testing.T) { metadata, err := client.WaitForOrchestrationCompletion(ctx, id) require.NoError(t, err) requireCompletedValue(t, metadata, 36) - require.GreaterOrEqual(t, tracker.releases.Load(), int32(1), "the idle timeout must have released a session") } // Scenario 9: entity operations across turns, a failed batch, and deletion. diff --git a/tests/durabletaskscheduler/dts_stress_test.go b/tests/durabletaskscheduler/dts_stress_test.go index f22056d8..c1f71ffe 100644 --- a/tests/durabletaskscheduler/dts_stress_test.go +++ b/tests/durabletaskscheduler/dts_stress_test.go @@ -404,22 +404,31 @@ func restartBackend(t *testing.T) error { if output, err := exec.Command("sh", "-c", command).CombinedOutput(); err != nil { return fmt.Errorf("restart command failed: %w: %s", err, output) } - client, err := durabletaskscheduler.NewClient(context.Background(), emulatorOptions(t), stressLogger()) - if err != nil { - return err - } - defer func() { _ = client.Close() }() + // NewClient sends a Hello, so it is part of the readiness check too. + var err error for deadline := time.Now().Add(3 * time.Minute); time.Now().Before(deadline); time.Sleep(500 * time.Millisecond) { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - _, err = client.FetchOrchestrationMetadata(ctx, api.InstanceID("stress-ready-"+uuid.NewString())) - cancel() - if err == nil || errors.Is(err, api.ErrInstanceNotFound) { + if err = backendAnswers(t); err == nil { return nil } } return fmt.Errorf("the backend did not answer after the restart: %w", err) } +func backendAnswers(t *testing.T) error { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + client, err := durabletaskscheduler.NewClient(ctx, emulatorOptions(t), stressLogger()) + if err != nil { + return err + } + defer func() { _ = client.Close() }() + _, err = client.FetchOrchestrationMetadata(ctx, api.InstanceID("stress-ready-"+uuid.NewString())) + if errors.Is(err, api.ErrInstanceNotFound) { + return nil + } + return err +} + // stressHarness owns one management client and a worker that can be // replaced while the test runs. type stressHarness struct { @@ -1068,9 +1077,11 @@ func verifyStressBatch(ctx context.Context, client *durabletaskscheduler.Client, if err != nil { if ctx.Err() != nil { status := "unknown" - if latest, fetchErr := client.FetchOrchestrationMetadata(context.Background(), batch.ids[i]); fetchErr == nil && latest != nil { + lookupCtx, cancelLookup := context.WithTimeout(context.Background(), 10*time.Second) + if latest, fetchErr := client.FetchOrchestrationMetadata(lookupCtx, batch.ids[i]); fetchErr == nil && latest != nil { status = latest.RuntimeStatus.String() } + cancelLookup() report("%s: not finished (%s): %v", batch.ids[i], status, err) return }